"""Action chunking and temporal ensembling, in NumPy. A chunking policy predicts the next k actions from one observation. Two ways to execute: open-loop chunks query every k steps and play the whole chunk temporal ensemble query every step; for time t, average every action that earlier chunks predicted for t, with weights w_i = exp(-m * i), where i = 0 is the oldest prediction (ACT paper, Algorithm 2) With m > 0 the oldest prediction gets the largest weight; m = 0 is a plain mean. Run as a script for the comparison from the lesson: python chunking.py """ from __future__ import annotations from collections import defaultdict from typing import Callable import numpy as np from numpy.typing import ArrayLike, NDArray ChunkPredictor = Callable[[int], NDArray[np.float64]] # time step -> (k, action_dim) chunk def ensemble_weights(n: int, m: float) -> NDArray[np.float64]: """Normalised weights for n predictions of one time step, oldest first.""" w = np.exp(-m * np.arange(n)) return w / w.sum() class TemporalEnsembler: """Online temporal ensembling of overlapping action chunks.""" def __init__(self, m: float) -> None: self.m = m self.buffer: defaultdict[int, list[NDArray[np.float64]]] = defaultdict(list) def step(self, t: int, chunk: ArrayLike) -> NDArray[np.float64]: """Store a chunk predicted at time t (actions for t, t+1, ...) and return the action for t.""" c = np.atleast_2d(np.asarray(chunk, dtype=float)) for j, a in enumerate(c): self.buffer[t + j].append(a) # appended in time order, so index 0 is the oldest preds = np.array(self.buffer.pop(t)) return ensemble_weights(len(preds), self.m) @ preds def run_open_loop_chunks(predict: ChunkPredictor, horizon: int, k: int) -> NDArray[np.float64]: """Query every k steps and execute each chunk in full.""" actions = [] for t in range(0, horizon, k): actions.extend(predict(t)[: min(k, horizon - t)]) return np.array(actions) def run_temporal_ensemble(predict: ChunkPredictor, horizon: int, m: float) -> NDArray[np.float64]: """Query every step and execute the temporally ensembled action.""" ens = TemporalEnsembler(m) return np.array([ens.step(t, predict(t)) for t in range(horizon)]) def step_change(actions: ArrayLike) -> tuple[float, float]: """Mean and largest absolute change between consecutive actions (smoothness measures).""" d = np.abs(np.diff(np.asarray(actions, dtype=float), axis=0)) return float(d.mean()), float(d.max()) def make_noisy_predictor( target: NDArray[np.float64], k: int, offset_std: float, noise_std: float, seed: int ) -> ChunkPredictor: """A stand-in for a learned policy: each chunk is the target plus one random offset per chunk and small per-step noise. Same t always gives the same chunk.""" def predict(t: int) -> NDArray[np.float64]: rng = np.random.default_rng([seed, t]) idx = np.minimum(np.arange(t, t + k), len(target) - 1) offset = rng.normal(0.0, offset_std) return (target[idx] + offset + rng.normal(0.0, noise_std, size=k))[:, None] return predict def main() -> None: horizon, k = 200, 20 t = np.arange(horizon) target = 0.5 * np.sin(2 * np.pi * t / horizon) # e.g. one joint angle in rad predict = make_noisy_predictor(target, k, offset_std=0.03, noise_std=0.005, seed=1) print(f"weights for 5 predictions, m = 0.1: {np.round(ensemble_weights(5, 0.1), 3)}") print(f"{'execution':<26}{'RMS error':>10}{'mean step':>11}{'max step':>10}") def report(name: str, a: NDArray[np.float64]) -> None: rms = np.sqrt(np.mean((a - target) ** 2)) mean_step, max_step = step_change(a) print(f"{name:<26}{rms:>10.4f}{mean_step:>11.4f}{max_step:>10.4f}") report("target", target) report(f"open-loop chunks, k = {k}", run_open_loop_chunks(predict, horizon, k)[:, 0]) for m in (0.01, 0.1): report(f"ensemble, m = {m}", run_temporal_ensemble(predict, horizon, m)[:, 0]) if __name__ == "__main__": main()