"""Behaviour cloning and DAgger on a 2D point-reaching task, in NumPy. The robot is a point in the plane. Each step it moves by the commanded action plus execution noise. The expert drives it from a start near (-1, 0) to the goal at (1, 0) along the upper half of the unit circle, so that it passes around an obstacle at the origin. Away from the arc, the expert also pulls the point back onto it. The demonstrations are recorded without execution noise, so they contain almost no recovery behaviour. This exaggerates what happens with careful human demonstrations. A small MLP policy (2 -> 32 tanh -> 2) learns the expert from position alone: behaviour cloning trains on expert demonstrations only DAgger runs the learner, asks the expert to label the states it visits, adds them to the dataset and retrains Run as a script for the comparison from the lesson: python reach.py """ from __future__ import annotations from dataclasses import dataclass, field from typing import Callable import numpy as np from numpy.typing import NDArray Policy = Callable[[NDArray[np.float64]], NDArray[np.float64]] START = np.array([-1.0, 0.0]) GOAL = np.array([1.0, 0.0]) HORIZON = 60 # steps per episode SPEED = 0.06 # expert step length along the arc GAIN = 0.5 # expert pull back towards the arc MAX_STEP = 0.1 # action limit (both expert and learner) START_NOISE = 0.05 # standard deviation of the start position DEMO_NOISE = 0.0 # execution noise while the expert demonstrates EXEC_NOISE = 0.02 # execution noise when a policy runs on the "real" system def clip_step(a: NDArray[np.float64]) -> NDArray[np.float64]: """Scale actions longer than MAX_STEP down to MAX_STEP. Works on (2,) or (N, 2).""" a = np.asarray(a, dtype=float) n = np.linalg.norm(a, axis=-1, keepdims=True) return a * np.minimum(1.0, MAX_STEP / np.maximum(n, 1e-12)) def expert(p: NDArray[np.float64]) -> NDArray[np.float64]: """Expert action for one position (2,) or a batch (N, 2). Follow the upper unit semicircle clockwise (from angle pi to 0) and pull radially towards radius 1. Near the goal, head straight for it. """ p = np.atleast_2d(np.asarray(p, dtype=float)) r = np.linalg.norm(p, axis=1, keepdims=True) radial = p / np.maximum(r, 1e-9) tangent = np.column_stack([radial[:, 1], -radial[:, 0]]) # clockwise a = SPEED * tangent + GAIN * (1.0 - r) * radial to_goal = GOAL - p near = np.linalg.norm(to_goal, axis=1) < 0.15 a[near] = to_goal[near] return clip_step(a).squeeze() def rollout( policy: Policy, rng: np.random.Generator, noise: float = EXEC_NOISE, horizon: int = HORIZON ) -> NDArray[np.float64]: """Run one episode and return the visited positions, shape (horizon + 1, 2).""" p = START + rng.normal(0.0, START_NOISE, size=2) path = [p.copy()] for _ in range(horizon): a = clip_step(policy(p)) p = p + a + rng.normal(0.0, noise, size=2) path.append(p.copy()) return np.array(path) def arc_error(path: NDArray[np.float64]) -> NDArray[np.float64]: """Distance of each position from the expert's arc (the upper unit semicircle).""" r = np.linalg.norm(path, axis=1) above = path[:, 1] >= 0.0 d_circle = np.abs(r - 1.0) # Below the x axis the closest point of the upper semicircle is one of its end points. d_ends = np.minimum(np.linalg.norm(path - START, axis=1), np.linalg.norm(path - GOAL, axis=1)) return np.where(above, d_circle, d_ends) @dataclass class MLP: """2 -> hidden (tanh) -> 2 network trained with full-batch Adam on mean squared error.""" hidden: int = 32 seed: int = 0 params: dict[str, NDArray[np.float64]] = field(init=False) def __post_init__(self) -> None: rng = np.random.default_rng(self.seed) self.params = { "W1": rng.normal(0.0, 1.0, size=(2, self.hidden)), "b1": np.zeros(self.hidden), "W2": rng.normal(0.0, 1.0 / np.sqrt(self.hidden), size=(self.hidden, 2)), "b2": np.zeros(2), } def __call__(self, x: NDArray[np.float64]) -> NDArray[np.float64]: h = np.tanh(np.atleast_2d(x) @ self.params["W1"] + self.params["b1"]) return (h @ self.params["W2"] + self.params["b2"]).squeeze() def fit(self, X: NDArray[np.float64], Y: NDArray[np.float64], epochs: int = 2000, lr: float = 0.01) -> float: """Minimise mean squared error with Adam. Returns the final training loss.""" P = self.params m = {k: np.zeros_like(v) for k, v in P.items()} v = {k: np.zeros_like(val) for k, val in P.items()} b1, b2, eps = 0.9, 0.999, 1e-8 n = len(X) loss = np.inf for t in range(1, epochs + 1): h = np.tanh(X @ P["W1"] + P["b1"]) out = h @ P["W2"] + P["b2"] err = out - Y loss = float(np.mean(np.sum(err**2, axis=1))) g_out = 2.0 * err / n dh = (g_out @ P["W2"].T) * (1.0 - h**2) grads = {"W2": h.T @ g_out, "b2": g_out.sum(0), "W1": X.T @ dh, "b1": dh.sum(0)} for k in P: m[k] = b1 * m[k] + (1 - b1) * grads[k] v[k] = b2 * v[k] + (1 - b2) * grads[k] ** 2 m_hat = m[k] / (1 - b1**t) v_hat = v[k] / (1 - b2**t) P[k] -= lr * m_hat / (np.sqrt(v_hat) + eps) return loss def collect(policy: Policy, n_episodes: int, rng: np.random.Generator, noise: float) -> NDArray[np.float64]: """Roll out a policy and return the visited states (without the final state).""" return np.vstack([rollout(policy, rng, noise)[:-1] for _ in range(n_episodes)]) def behaviour_cloning(n_demos: int, seed: int = 0, epochs: int = 2000) -> MLP: """Train on n_demos expert demonstrations.""" rng = np.random.default_rng(seed) X = collect(expert, n_demos, rng, DEMO_NOISE) net = MLP(seed=seed) net.fit(X, expert(X), epochs=epochs) return net def dagger( n_demos: int, n_iters: int, rollouts_per_iter: int, seed: int = 0, epochs: int = 2000 ) -> tuple[MLP, list[float]]: """DAgger with beta_1 = 1 (the first policy is plain behaviour cloning) and beta_i = 0 after. Returns the final policy and the mean arc error of the policy after each round (round 0 is the behaviour-cloning policy). """ rng = np.random.default_rng(seed) X = collect(expert, n_demos, rng, DEMO_NOISE) net = MLP(seed=seed) net.fit(X, expert(X), epochs=epochs) history = [evaluate(net)["arc_error"]] for _ in range(n_iters): X_new = collect(net, rollouts_per_iter, rng, EXEC_NOISE) # states the learner visits X = np.vstack([X, X_new]) # aggregate; every state is labelled by the expert below net.fit(X, expert(X), epochs=epochs) # warm start from the previous weights history.append(evaluate(net)["arc_error"]) return net, history def evaluate(policy: Policy, n_episodes: int = 50, seed: int = 123) -> dict[str, float]: """Mean arc error over all steps, mean final distance to goal, and success rate (< 0.1 m).""" rng = np.random.default_rng(seed) paths = [rollout(policy, rng) for _ in range(n_episodes)] final = np.array([np.linalg.norm(p[-1] - GOAL) for p in paths]) return { "arc_error": float(np.mean([arc_error(p).mean() for p in paths])), "final_dist": float(final.mean()), "success": float(np.mean(final < 0.1)), } def action_error(policy: Policy, states: NDArray[np.float64]) -> float: """Mean distance between the policy's actions and the expert's actions on given states.""" pred = np.atleast_2d(clip_step(policy(states))) return float(np.mean(np.linalg.norm(pred - expert(states), axis=1))) def distribution_shift(policy: Policy, n_episodes: int = 20, seed: int = 7) -> tuple[float, float]: """Action error on clean expert demonstration states and on the policy's own states.""" rng = np.random.default_rng(seed) demo_states = collect(expert, n_episodes, rng, DEMO_NOISE) own_states = collect(policy, n_episodes, rng, EXEC_NOISE) return action_error(policy, demo_states), action_error(policy, own_states) def main() -> None: n_demos, n_iters, per_iter = 5, 6, 5 total = n_demos + n_iters * per_iter print(f"{'policy':<34}{'arc error':>10}{'final dist':>12}{'success':>9}") def report(name: str, policy: Policy) -> None: r = evaluate(policy) print(f"{name:<34}{r['arc_error']:>10.3f}{r['final_dist']:>12.3f}{r['success']:>9.2f}") report("expert", expert) report(f"BC, {n_demos} demos", behaviour_cloning(n_demos)) bc = behaviour_cloning(total) report(f"BC, {total} demos", bc) dag, history = dagger(n_demos, n_iters, per_iter) report(f"DAgger, {n_demos} demos + {n_iters} x {per_iter} rollouts", dag) print("DAgger arc error per round:", " ".join(f"{e:.3f}" for e in history)) for name, policy in (("BC", bc), ("DAgger", dag)): on_demo, on_own = distribution_shift(policy) print(f"{name} action error: {on_demo:.4f} on demo states, {on_own:.4f} on its own states") if __name__ == "__main__": main()