Add eval_kit evaluation suite with WebRTC visualization

- eval_kit/core: 零框架依赖的指标内核（Trace/metrics/aggregation/terrain_spec/report/naming）
- eval_kit/isaaclab: 适配层（env_cfg 改造、rollout 采集、checkpoint 加载）
- eval_kit/profiles.py / protocols.py: 机器人形态描述与评估口径预设
- eval_kit/tests: 24 项内核自检 + 21 项命令行自检（都不需要仿真）
- eval_suite.py: 编排入口，每个（阶段, 种子）一个子进程；与 eval.py 指标口径一致
- WebRTC 可视化：--livestream 透传、视口相机跟随/自由（--view-env / --free-camera）、
  连接前停顿（--pause-s）、自动放开客户端改画幅、打印连接地址
- 诊断与替代路径：rollout 进度打印、--record-frames 抓帧

scripts/reinforcement_learning/rsl_rl/eval_suite.py

@@ -0,0 +1,893 @@
+# Copyright (c) 2025 Deep Robotics
+# SPDX-License-Identifier: BSD 3-Clause
+
+"""评价套件的命令行入口。
+
+与 ``eval.py`` 并存、互不影响：本脚本不导入也不修改 ``eval.py``，两者可以并行使用，
+输出 JSON 结构相似（指标名完全一致），便于互相对照。
+
+用法示例::
+
+    # 标准评估（默认协议与火蚁描述）
+    python scripts/reinforcement_learning/rsl_rl/eval_suite.py \\
+        --task Rough-Deeprobotics-M20-v0 \\
+        --checkpoint /绝对路径/model_19999.pt \\
+        --headless
+
+    # 上限评估，5 个随机种子，顺带把轨迹存下来
+    python scripts/reinforcement_learning/rsl_rl/eval_suite.py \\
+        --task Rough-Deeprobotics-M20-v0 \\
+        --checkpoint /绝对路径/model_19999.pt \\
+        --protocol standard_v1_multiseed --save-trace /tmp/traces --headless
+
+    # 抗扰评估
+    python scripts/reinforcement_learning/rsl_rl/eval_suite.py \\
+        --task Rough-Deeprobotics-M20-v0 \\
+        --checkpoint /绝对路径/model_19999.pt \\
+        --eval-type robustness --headless
+
+    # 用 WebRTC 看画面（不需要显示器；局域网内用 Isaac Sim WebRTC Streaming Client 连）
+    python scripts/reinforcement_learning/rsl_rl/eval_suite.py \\
+        --task Rough-Deeprobotics-M20-v0 \\
+        --checkpoint /绝对路径/model_19999.pt \\
+        --eval-type upper_limits --terrain-type stairs_up \\
+        --livestream 2
+
+    ``--livestream 1`` 走公网（NVIDIA NVCF 信令，需要 ``PUBLIC_IP``），
+    ``--livestream 2`` 走局域网直连；两者都由 ``AppLauncher`` 提供，详见
+    ``isaaclab.app.AppLauncher`` 的文档。本脚本只额外保证：视口相机跟着被评估的
+    机器人（``--view-env``）、开始 rollout 前留出连接时间（``--pause-s``）、
+    以及把「客户端该连哪里」打印出来。
+
+    协议预设与机器人描述都在 ``eval_kit/`` 里：改口径改 ``protocols.py``，
+    加机器人改 ``profiles.py``。
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from dataclasses import replace
+from typing import Any
+
+#: 原始命令行，用于在需要多个环境时 fork 子进程（见文件末尾的说明）。
+_ORIGINAL_ARGV = list(sys.argv[1:])
+
+# --- 路径引导：必须在导入 eval_kit / rl_training 之前完成 --------------------------
+_HERE = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.abspath(os.path.join(_HERE, "..", "..", ".."))
+PROJECT_SOURCE_PATH = os.path.join(PROJECT_ROOT, "source", "rl_training")
+for path in (PROJECT_SOURCE_PATH, PROJECT_ROOT):
+    if path not in sys.path:
+        sys.path.insert(0, path)
+
+from isaaclab.app import AppLauncher  # noqa: E402
+
+from eval_kit.core.report import (  # noqa: E402
+    EVAL_TYPES,
+    filter_metrics,
+    format_aggregate_summary,
+    format_metric_result,
+    format_rollout_summary,
+    format_upper_limit_summary,
+)
+from eval_kit.core.schema import EvalProtocol  # noqa: E402
+from eval_kit.core.terrain_spec import get_terrain_spec  # noqa: E402
+from eval_kit.profiles import PROFILES, get_profile  # noqa: E402
+from eval_kit.protocols import PROTOCOLS, get_protocol  # noqa: E402
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="基于 eval_kit 的机器人性能评估（与 eval.py 并存，指标口径一致）。",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("--task", type=str, default="Rough-Deeprobotics-M20-v0", help="任务名。")
+    parser.add_argument(
+        "--agent", type=str, default="rsl_rl_cfg_entry_point", help="agent 配置入口名（配合 Hydra）。"
+    )
+    parser.add_argument(
+        "--profile", type=str, default="solenopsis", choices=sorted(PROFILES), help="机器人形态描述。"
+    )
+    parser.add_argument(
+        "--protocol", type=str, default="standard_v1", choices=sorted(PROTOCOLS), help="评估协议预设。"
+    )
+    parser.add_argument(
+        "--eval-type", dest="eval_type", type=str, default="all", choices=list(EVAL_TYPES), help="评估类型。"
+    )
+
+    checkpoint = parser.add_argument_group("checkpoint")
+    checkpoint.add_argument("--checkpoint", type=str, default=None, help="checkpoint 路径（支持绝对路径）。")
+    checkpoint.add_argument("--load-run", dest="load_run", type=str, default=None, help="运行目录或目录名。")
+    checkpoint.add_argument(
+        "--experiment-name", dest="experiment_name", type=str, default=None,
+        help="实验名，用于在 logs/rsl_rl/<实验名> 下查找；不传则用 agent 配置里的。",
+    )
+
+    run = parser.add_argument_group("运行规模（不传则取协议预设值）")
+    run.add_argument(
+        "--num-envs", dest="num_envs", type=int, default=250,
+        help="并行环境数（上限评估时会按网格规模自动覆盖）。",
+    )
+    # 注意：--device 由 AppLauncher.add_app_launcher_args() 提供，这里不能再加。
+    # 它保留了 device / headless / enable_cameras / experience / livestream /
+    # rendering_mode / xr 这几个名字，重复定义会直接抛 ValueError。
+    run.add_argument(
+        "--seeds", type=int, nargs="+", default=None,
+        help="重复试验的随机种子，覆盖协议预设；给多个种子时会输出 mean±std。",
+    )
+    run.add_argument("--save-trace", dest="save_trace", type=str, default=None, help="把轨迹存到该目录。")
+    run.add_argument(
+        "--view-env", dest="view_env", type=int, default=None,
+        help="让视口相机跟随第 N 个环境的机器人（非 headless 或开启推流时有效）。"
+             "不传时默认跟随 env 0。环境编号 0 起，"
+             "编号与（地形, 难度）的对应关系会在启动时打印。",
+    )
+    run.add_argument(
+        "--free-camera", dest="free_camera", action="store_true", default=False,
+        help="不要让视口相机跟随任何机器人：起始摆在俯视全场的位置，之后在 WebRTC 客户端里"
+             "自己移动（脚本与 IsaacLab 都不会再改写相机）。同时给了 --view-env 时以本参数为准。",
+    )
+    run.add_argument(
+        "--pause-s", dest="pause_s", type=float, default=None,
+        help="开始 rollout 之前停顿的秒数：期间画面照常推流、物理冻结，按回车可提前开始。"
+             "用来给 WebRTC 客户端留连接时间。开启推流时默认 30 s，其余情况默认 0（不停顿）。",
+    )
+    run.add_argument(
+        "--record-frames", dest="record_frames", type=str, default=None,
+        help="把 rollout 期间的视口画面按帧存成 PNG 到该目录（例如 frame_000150.png）。"
+             "推流传输不可用时用它离线看评估过程；抓帧间隔见 --record-every。",
+    )
+    run.add_argument(
+        "--record-every", dest="record_every", type=int, default=5,
+        help="--record-frames 每多少步抓一帧。",
+    )
+
+    overrides = parser.add_argument_group("口径覆盖（不传则取协议预设值，改动会反映到 fingerprint）")
+    overrides.add_argument("--episode-length-s", dest="episode_length_s", type=float, default=None)
+    overrides.add_argument("--command-x", dest="command_x", type=float, default=None)
+    overrides.add_argument("--command-y", dest="command_y", type=float, default=None)
+    overrides.add_argument("--command-yaw", dest="command_yaw", type=float, default=None)
+    overrides.add_argument("--start-x", dest="start_x", type=float, default=None)
+    overrides.add_argument("--start-y", dest="start_y", type=float, default=None)
+    overrides.add_argument(
+        "--start-z-offset", dest="start_z_offset", type=float, default=None,
+        help="起始高度相对地形瓦片中心的抬升 [m]。倒金字塔台阶这类「中心最低」的地形需要正值，"
+             "否则机器人会被生成在地形内部（默认 0.0，与旧 eval.py 一致）。",
+    )
+    overrides.add_argument("--terrain-size", dest="terrain_size", type=float, default=None, help="地形瓦片边长 [m]。")
+    overrides.add_argument("--success-distance", dest="success_distance", type=float, default=None)
+    overrides.add_argument("--tilt-threshold", dest="tilt_threshold", type=float, default=None, help="摔倒倾角 [rad]。")
+    overrides.add_argument(
+        "--acc-threshold-g", dest="acc_threshold_g", type=float, default=None,
+        help="不稳定判定的加速度阈值，单位 g。",
+    )
+    overrides.add_argument(
+        "--energy-convention", dest="energy_convention", type=str, default=None, choices=["absolute", "signed"]
+    )
+    overrides.add_argument(
+        "--robot-mass", dest="robot_mass", type=float, default=None,
+        help="覆盖 COT 用质量；默认从资产自动读取。",
+    )
+    overrides.add_argument(
+        "--terrain-type", dest="terrain_type", type=str, default="all",
+        help="上限评估只用某一种地形：all 或地形规格里的类型名。",
+    )
+    overrides.add_argument("--num-levels", dest="num_levels", type=int, default=None, help="上限评估的难度等级数。")
+    overrides.add_argument("--envs-per-cell", dest="envs_per_cell", type=int, default=None, help="每格机器人数。")
+    overrides.add_argument(
+        "--no-terrain-bounds", dest="enforce_terrain_bounds", action="store_false", default=None,
+        help="去掉越界终止项。地砖小于 3.8 m 时它会误判最外侧格子（详见 --help 里 protocol 的说明）。",
+    )
+
+    output = parser.add_argument_group("输出")
+    output.add_argument("--output", type=str, default=None, help="输出 JSON 路径；默认写在 checkpoint 同目录的 eval_kit/ 下。")
+    output.add_argument("--print-json", dest="print_json", action="store_true", default=False, help="把 JSON 打到 stdout。")
+    # 不能叫 --verbose：AppLauncher.add_app_launcher_args() 已经占用了这个名字，
+    # 重复定义会让 argparse 直接抛 ArgumentError（旧 eval.py 就卡在这里）。
+    output.add_argument(
+        "--verbose-terrain", dest="verbose_terrain", action="store_true", default=False,
+        help="打印逐格地形/难度调试信息。（注意 --verbose 属于 AppLauncher，行为不同）",
+    )
+    output.add_argument(
+        "--include-summary", dest="include_summary", action="store_true", default=False,
+        help="eval-type=all 时额外跑一次混合地形汇总。",
+    )
+    # 内部参数：一个进程只能建一个环境，需要多个环境时由父进程 fork 自己并传这些参数。
+    # 不建议手工使用。
+    internal = parser.add_argument_group("内部（子进程用，勿手工指定）")
+    internal.add_argument("--stage", type=str, default="", choices=["", "upper", "plain"])
+    internal.add_argument("--stage-seed", dest="stage_seed", type=int, default=0)
+    internal.add_argument("--stage-out", dest="stage_out", type=str, default=None)
+    return parser
+
+
+def _apply_overrides(protocol: EvalProtocol, args: argparse.Namespace) -> EvalProtocol:
+    """把命令行显式给出的口径覆盖到协议预设上。没给的不动，保持预设原值。"""
+    updates: dict[str, object] = {}
+    if args.episode_length_s is not None:
+        updates["steps"] = int(round(args.episode_length_s / protocol.dt))
+    if args.command_x is not None:
+        updates["command"] = (args.command_x, protocol.command[1], protocol.command[2])
+    if args.command_y is not None:
+        current = updates.get("command", protocol.command)
+        updates["command"] = (current[0], args.command_y, current[2])
+    if args.command_yaw is not None:
+        current = updates.get("command", protocol.command)
+        updates["command"] = (current[0], current[1], args.command_yaw)
+    if args.start_x is not None:
+        updates["start_pose"] = (args.start_x, protocol.start_pose[1], protocol.start_pose[2])
+    if args.start_y is not None:
+        current = updates.get("start_pose", protocol.start_pose)
+        updates["start_pose"] = (current[0], args.start_y, current[2])
+    if args.start_z_offset is not None:
+        updates["start_z_offset_m"] = args.start_z_offset
+    if args.terrain_size is not None:
+        updates["terrain_size_m"] = args.terrain_size
+    if args.success_distance is not None:
+        updates["success_distance_m"] = args.success_distance
+    if args.tilt_threshold is not None:
+        updates["tilt_threshold_rad"] = args.tilt_threshold
+    if args.acc_threshold_g is not None:
+        updates["acc_threshold_g"] = args.acc_threshold_g
+    if args.energy_convention is not None:
+        updates["energy_convention"] = args.energy_convention
+    if args.num_levels is not None:
+        updates["num_levels"] = args.num_levels
+    if args.envs_per_cell is not None:
+        updates["envs_per_cell"] = args.envs_per_cell
+    if args.seeds is not None:
+        updates["seeds"] = tuple(args.seeds)
+    if args.enforce_terrain_bounds is not None:
+        updates["enforce_terrain_bounds"] = args.enforce_terrain_bounds
+    return replace(protocol, **updates)  # type: ignore[arg-type]
+
+
+def _terrain_types_for(args: argparse.Namespace, spec) -> list[str]:
+    if args.terrain_type == "all":
+        return list(spec.types)
+    if args.terrain_type not in spec.types:
+        raise SystemExit(
+            f"地形类型 {args.terrain_type!r} 不在规格 {spec.name!r} 里；可选：{list(spec.types)}"
+        )
+    return [args.terrain_type]
+
+
+def _resolve_output_path(args: argparse.Namespace, resume_path: str, eval_type: str) -> str:
+    if args.output is not None:
+        path = os.path.abspath(os.path.expanduser(args.output))
+    else:
+        from datetime import datetime
+
+        directory = os.path.join(os.path.dirname(resume_path), "eval_kit")
+        suffix = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        path = os.path.join(directory, f"{eval_type}_{suffix}.json")
+    parent = os.path.dirname(path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    return path
+
+
+def _livestream_value(args_cli: argparse.Namespace) -> int:
+    """解析实际生效的推流模式：0 关、1 公网 WebRTC、2 局域网 WebRTC。
+
+    ``--livestream`` 的默认值是 -1，含义是「听 ``LIVESTREAM`` 环境变量的」，
+    所以 -1 时要回退去读环境变量，而不是当成 0。
+    """
+    try:
+        value = int(getattr(args_cli, "livestream", -1))
+    except (TypeError, ValueError):
+        value = -1
+    if value < 0:
+        try:
+            value = int(os.environ.get("LIVESTREAM", 0) or 0)
+        except ValueError:
+            value = 0
+    return value
+
+
+def _display_enabled(args_cli: argparse.Namespace) -> bool:
+    """这次运行会不会给用户看画面（本地 GUI 或推流）。"""
+    return not getattr(args_cli, "headless", True) or _livestream_value(args_cli) > 0
+
+
+def _viewer_env_index(args_cli: argparse.Namespace) -> int | None:
+    """视口相机跟随哪个环境；不跟随时（自由视角、或完全不需要画面）返回 None。
+
+    必须在构造 ``AppLauncher`` **之前**调用：``AppLauncher`` 会把 ``livestream``
+    从 namespace 里 pop 掉，还会在开启推流时把 ``headless`` 改写成 True（推流在
+    服务端本来就按 headless 启动），等它跑完再读这两个值就分不清「用户要不要看画面」了。
+
+    推流时 ``SimulationContext`` 的 ``has_gui()``（本地 GUI 或推流 GUI）仍为真、
+    ``env_cfg.viewer`` 也仍然生效，所以推流和本地 GUI 两种情况都要设视口相机，
+    否则推流画面停在默认相机上、看不到机器人。
+    """
+    if not _display_enabled(args_cli) or getattr(args_cli, "free_camera", False):
+        return None
+    return 0 if args_cli.view_env is None else int(args_cli.view_env)
+
+
+def _set_free_camera_overview(env: Any) -> None:
+    """自由视角模式：把相机摆到能看见整片地形的位置。
+
+    ``env_cfg.viewer.origin_type`` 是 ``world`` 时，``ViewportCameraController``
+    只在初始化时设一次相机、之后不再改写，所以这里设的视角会一直保留，
+    客户端里可以随便移动鼠标/键盘来观察。
+    """
+    import numpy as np
+
+    origins = env.unwrapped.scene.env_origins
+    center = origins.mean(dim=0)
+    radius = float((origins - center).norm(dim=1).max().item())
+    center_np = center.detach().cpu().numpy()
+    eye = center_np + np.array([0.0, -1.6 * radius, 1.25 * radius], dtype=center_np.dtype)
+    env.unwrapped.sim.set_camera_view(eye=eye, target=center_np)
+    print(
+        f"[eval_kit] 自由视角：相机不跟随机器人，起始俯视全场"
+        f"（地形中心 {np.round(center_np, 1).tolist()}，半径 {radius:.1f} m），"
+        f"之后在客户端里自己移动",
+        flush=True,
+    )
+
+
+def _allow_client_resize(args_cli: argparse.Namespace) -> None:
+    """让 WebRTC 客户端可以改画面尺寸（默认关掉会冻住画面）。
+
+    客户端一旦改窗口大小/全屏，就会发 ``onCustomResizeMessage``；服务端默认拒绝
+    （``... must both be positive and /app/livestream/allowDynamicResize enabled``），
+    同时推流用的 ``shared swapchain buffer`` 进入坏状态 —— 之后画面不再更新，而仿真
+    照常在跑（终端里的 rollout 进度还在涨）。这两个设置通过 ``--kit_args`` 传给 Kit。
+
+    用户自己显式给了同名设置就不覆盖（想关掉就传 ``--kit_args=--/app/livestream/allowDynamicResize=false``）。
+    """
+    wanted = ("--/app/livestream/allowResize=true", "--/app/livestream/allowDynamicResize=true")
+    existing = args_cli.kit_args or ""
+    extra = [flag for flag in wanted if flag.split("=")[0] not in existing]
+    if not extra:
+        return
+    args_cli.kit_args = " ".join(filter(None, [existing, *extra]))
+    print(
+        f"[eval_kit] 推流追加 kit 设置：{' '.join(extra)}（允许客户端调整窗口大小/全屏）",
+        flush=True,
+    )
+
+
+def _grab_viewport_frame(state: dict, directory: str, every: int) -> None:
+    """每 ``every`` 步向视口要一张 PNG。
+
+    抓帧是异步的（等下一次渲染产出），所以这里不阻塞 rollout：上一张还没写完就跳过
+    这一帧。写完的文件名形如 ``frame_000150.png``，可用 ffmpeg 合成视频。
+    """
+    if every <= 0 or state["step"] % every:
+        return
+    pending = state["pending"]
+    if pending is not None and not os.path.exists(pending):
+        return
+    from omni.kit.viewport.utility import capture_viewport_to_file, get_active_viewport
+
+    path = os.path.join(directory, f"frame_{state['step']:06d}.png")
+    state["pending"] = path
+    capture_viewport_to_file(get_active_viewport(), path)
+
+
+def _with_frame_capture(policy: Any, directory: str, every: int) -> Any:
+    """把策略包一层，顺带按间隔抓视口帧（策略每被调用一次就是 rollout 推进一步）。"""
+    os.makedirs(directory, exist_ok=True)
+    state: dict = {"step": 0, "pending": None}
+    print(f"[eval_kit] rollout 期间抓帧到 {directory}（每 {every} 步一张 PNG）", flush=True)
+
+    def wrapped(obs: Any) -> Any:
+        state["step"] += 1
+        _grab_viewport_frame(state, directory, every)
+        return policy(obs)
+
+    return wrapped
+
+
+def _report_recorded_frames(directory: str, every: int, dt: float) -> None:
+    """抓帧结束后提示怎么合成视频（帧是按仿真时间等间隔抓的，fps = 1/(every*dt)）。"""
+    frames = sorted(name for name in os.listdir(directory) if name.endswith(".png"))
+    if not frames:
+        print(f"[eval_kit] 抓帧目录 {directory} 里没有帧（视口没渲染？）", flush=True)
+        return
+    fps = 1.0 / (every * dt) if every > 0 and dt > 0 else 10.0
+    print(f"[eval_kit] 抓到 {len(frames)} 帧：{directory}", flush=True)
+    print(
+        f"[eval_kit]   合成视频（按仿真速度播放）：cd {directory} && "
+        f"ffmpeg -y -framerate {fps:.2f} -pattern_type glob -i '*.png' "
+        f"-pix_fmt yuv420p -c:v libx264 -vf 'pad=ceil(iw/2)*2:ceil(ih/2)*2' /tmp/eval.mp4",
+        flush=True,
+    )
+
+
+def _local_ipv4_addresses() -> list[str]:
+    """本机的非回环 IPv4 地址，用来告诉用户 WebRTC 客户端该连哪里。"""
+    import socket
+
+    addresses: list[str] = []
+    try:
+        for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET):
+            address = info[4][0]
+            if address not in addresses:
+                addresses.append(address)
+    except OSError:
+        pass
+    try:
+        # 不真的发包：只是让内核按路由表挑一个默认出口地址
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
+            probe.connect(("8.8.8.8", 80))
+            address = probe.getsockname()[0]
+            if address not in addresses:
+                addresses.append(address)
+    except OSError:
+        pass
+    return [address for address in addresses if not address.startswith("127.")]
+
+
+def _print_livestream_info(livestream: int) -> None:
+    """打印客户端该连到哪里。要在 Kit 起来之后调用，端口从 carb 设置里读实际值。"""
+    import carb.settings
+
+    port = carb.settings.get_settings().get("/app/livestream/port")
+    addresses = _local_ipv4_addresses()
+    if livestream == 1:
+        public_ip = os.environ.get("PUBLIC_IP") or (addresses[0] if addresses else "127.0.0.1")
+        print(f"[eval_kit] WebRTC 推流已启用（--livestream 1，公网，PUBLIC_IP={public_ip}）", flush=True)
+    else:
+        print("[eval_kit] WebRTC 推流已启用（--livestream 2，局域网直连）", flush=True)
+    if addresses:
+        targets = "  ".join(f"{address}:{port}" for address in addresses)
+        print(f"[eval_kit]   客户端连接地址：{targets}", flush=True)
+    else:
+        print(f"[eval_kit]   客户端连接地址：<本机 IP>:{port}", flush=True)
+    print(
+        "[eval_kit]   本子进程结束后推流会断开（每个阶段/种子都是一个新进程），"
+        "客户端重连即可",
+        flush=True,
+    )
+
+
+def _hold_before_rollout(sim: Any, seconds: float, *, start_height_note: str) -> None:
+    """停顿 ``seconds`` 秒再开始 rollout：画面继续推流、物理冻结。
+
+    用 ``sim.render()`` 而不是 ``time.sleep``：独立进程模式下 Kit 的更新循环由我们
+    调用 ``app.update()`` 驱动，光 sleep 推流画面会冻住。``render()`` 内部会先把
+    ``/app/player/playSimulations`` 关掉再 update，因此画面在刷新、时间不前进。
+    另外把 timeline 暂停，避免物理在等待期间自己走掉（那会让评估起点不是刚重置的状态）。
+    """
+    import select
+    import time as _time
+
+    sim.pause()
+    started = _time.monotonic()
+    deadline = started + seconds
+    # 明确说明「悬空不动」是正常的：这一窗物理是冻结的，机器人正停在法规的起始位姿上
+    print(
+        f"[eval_kit] 画面已就绪，先停 {seconds:.0f} s 再开始 rollout —— 期间物理是冻结的，"
+        f"机器人在起始位姿上不动{start_height_note}，看起来「悬空不动」是正常的。",
+        flush=True,
+    )
+    if sys.stdin is not None and sys.stdin.isatty():
+        print("[eval_kit]   在终端按回车可立即开始 rollout", flush=True)
+    next_report = started + 5.0
+    while _time.monotonic() < deadline:
+        sim.render()
+        if sys.stdin is not None and sys.stdin.isatty() and select.select([sys.stdin], [], [], 0)[0]:
+            sys.stdin.readline()
+            break
+        if _time.monotonic() >= next_report:
+            print(f"[eval_kit]   剩余 {max(0.0, deadline - _time.monotonic()):.0f} s（物理仍冻结）", flush=True)
+            next_report += 5.0
+    sim.play()
+    print(f"[eval_kit] 停顿 {_time.monotonic() - started:.1f} s 结束，开始 rollout（画面从这里才动）", flush=True)
+
+
+def _format_early_reset_warning(early: dict, protocol: EvalProtocol, min_terrain_size: float) -> str:
+    """把早退统计变成可执行的建议。"""
+    import json as _json
+
+    lines = [
+        "[警告] 有环境在 episode 中途被重置，路程/进度类指标可能被污染："
+        f"{_json.dumps(early, ensure_ascii=False)}"
+    ]
+    if protocol.enforce_terrain_bounds:
+        lines.append("  最常见的原因是 IsaacLab 的 terrain_out_of_bounds 终止项：它比较机器人的")
+        lines.append("  世界坐标与整张地图的范围，而不是机器人自己那块地砖，于是最外侧的格子")
+        lines.append(f"  从第 0 步起就被判越界。当前地砖 {protocol.terrain_size_m} m，")
+        lines.append(f"  要保留该终止项需要 >= {min_terrain_size:.1f} m。三种处置：")
+        lines.append(f"    1) --terrain-size {max(min_terrain_size, protocol.terrain_size_m):.1f}  保留终止项，消除误判")
+        lines.append("    2) --no-terrain-bounds         去掉该终止项")
+        lines.append("    3) --protocol standard_v1_clean  等价于方案 1")
+    return "\n".join(lines)
+
+
+def _stage_argv(stage: str, seed: int, out_path: str) -> list[str]:
+    """构造子进程命令行：在原始参数上追加阶段参数。"""
+    return [
+        sys.executable,
+        os.path.abspath(__file__),
+        *_ORIGINAL_ARGV,
+        "--stage", stage,
+        "--stage-seed", str(seed),
+        "--stage-out", out_path,
+    ]
+
+
+def _spawn_stage(stage: str, seed: int) -> dict[str, object]:
+    """在子进程里跑一个「一个环境」的阶段，读回它写的 JSON。
+
+    为什么必须开子进程：IsaacLab 在本配置下一个进程只能创建一个环境。第一个环境
+    建好之后，``ROUGH_TERRAINS_CFG`` 这类模块级单例会被运行时长出自引用，之后
+    任何 cfg 构造（新实例 ``env_cfg_cls()``、开场存的 pristine 深拷贝、重新深拷贝）
+    都会在 ``cfg.validate()`` 里 RecursionError。三种补救方式都实测失败，所以只能
+    一个环境一个进程。
+    """
+    handle = tempfile.NamedTemporaryFile(prefix=f"eval_kit_{stage}_", suffix=".json", delete=False)
+    handle.close()
+    out_path = handle.name
+    argv = _stage_argv(stage, seed, out_path)
+    print(f"[eval_kit] 子进程 stage={stage} seed={seed}", flush=True)
+    completed = subprocess.run(argv)
+    if completed.returncode != 0:
+        os.unlink(out_path)
+        raise SystemExit(f"子进程 stage={stage} seed={seed} 失败（退出码 {completed.returncode}）")
+    try:
+        with open(out_path, encoding="utf-8") as file_handle:
+            return json.load(file_handle)
+    finally:
+        os.unlink(out_path)
+
+
+def _recorder_report(recorder) -> dict[str, object]:
+    return {
+        "feet": recorder.feet_names if recorder else [],
+        "support": recorder.support_names if recorder else [],
+        "joints": recorder.joint_names if recorder else [],
+        "mass_kg": recorder.mass_kg if recorder else None,
+        "warnings": recorder.warnings if recorder else [],
+    }
+
+
+def _run_parent(args_cli: argparse.Namespace) -> None:
+    """父进程：不启动仿真，只做编排与合并。每个环境交给一个子进程。"""
+    import numpy as np
+
+    from eval_kit.core.aggregation import aggregate_runs, build_upper_limit_matrix
+    from eval_kit.core.report import filter_metrics, format_aggregate_summary, format_metric_result
+    from eval_kit.core.report import format_rollout_summary, format_upper_limit_summary
+
+    profile = get_profile(args_cli.profile)
+    protocol = _apply_overrides(get_protocol(args_cli.protocol), args_cli)
+    spec = get_terrain_spec(protocol.terrain_spec)
+    if args_cli.robot_mass is not None:
+        profile = replace(profile, mass_kg=args_cli.robot_mass)
+
+    if not args_cli.checkpoint:
+        raise SystemExit(
+            "编排进程需要 --checkpoint 才能派发子进程（父进程不加载 hydra，"
+            "因此无法用 --load-run 按实验名查找）。请传入 checkpoint 路径。"
+        )
+    resume_path = os.path.abspath(os.path.expanduser(args_cli.checkpoint))
+    if not os.path.isfile(resume_path):
+        raise SystemExit(f"找不到 checkpoint：{resume_path}")
+
+    print("[eval_kit] 协议预设:", args_cli.protocol)
+    print("[eval_kit] profile   :", profile.name)
+    print("[eval_kit] checkpoint:", resume_path)
+    print("[eval_kit] 口径      :", json.dumps(protocol.to_dict(), ensure_ascii=False))
+    print("[eval_kit] 地形规格  :", spec.to_dict())
+    print("[eval_kit] 种子      :", list(protocol.seeds), "（每个种子一个子进程）")
+
+    result: dict[str, object] = {
+        "suite": "eval_kit",
+        "task": args_cli.task,
+        "eval_type": args_cli.eval_type,
+        "checkpoint": resume_path,
+        "profile": profile.to_dict(),
+        "protocol": protocol.to_dict(),
+        "terrain_spec": spec.to_dict(),
+        "seeds": list(protocol.seeds),
+        "runner": "one-env-per-process",
+    }
+
+    wants_upper = args_cli.eval_type in ("upper_limits", "all")
+
+    if wants_upper:
+        terrain_types = _terrain_types_for(args_cli, spec)
+        per_seed: list[dict[str, object]] = []
+        pooled: dict[str, list[float]] = {"success": [], "max_progress": [], "max_tilt": []}
+        pooled_levels: list[np.ndarray] = []
+        pooled_types: list[np.ndarray] = []
+        recorder_report: dict[str, object] = {}
+
+        for seed in protocol.seeds:
+            data = _spawn_stage("upper", seed)
+            recorder_report = data["recorder"]  # type: ignore[assignment]
+            per_env = data["per_env"]  # type: ignore[assignment]
+            level_ids = np.asarray(data["level_ids"])  # type: ignore[arg-type]
+            type_ids = np.asarray(data["type_ids"])  # type: ignore[arg-type]
+            for key in pooled:
+                pooled[key].extend(per_env[key])  # type: ignore[index]
+            pooled_levels.append(level_ids)
+            pooled_types.append(type_ids)
+            matrix = build_upper_limit_matrix(
+                per_env, level_ids, type_ids, terrain_types, spec, protocol.num_levels  # type: ignore[arg-type]
+            )
+            per_seed.append(
+                {
+                    "seed": seed,
+                    "scores": matrix["scores"],
+                    "matrix": matrix["matrix"],
+                    "metrics": data["metrics"],
+                    "early_resets": data["early_resets"],
+                }
+            )
+            if args_cli.verbose_terrain:
+                print(f"[eval_kit] seed {seed}: " + "  ".join(f"{k}={v:.2f}" for k, v in matrix["scores"].items()))
+
+        pooled_matrix = build_upper_limit_matrix(
+            pooled, np.concatenate(pooled_levels), np.concatenate(pooled_types), terrain_types, spec, protocol.num_levels
+        )
+        result["terrain_types"] = terrain_types
+        result["upper_limits"] = pooled_matrix
+        result["upper_limits_per_seed"] = per_seed
+        result["recorder"] = recorder_report
+        print()
+        print(format_upper_limit_summary(pooled_matrix))
+        first_metrics = per_seed[0]["metrics"]  # type: ignore[index]
+        print()
+        print(format_rollout_summary(first_metrics))  # type: ignore[arg-type]
+
+        for entry in per_seed:
+            early = entry["early_resets"]  # type: ignore[index]
+            if early and early["polluted"]:  # type: ignore[index]
+                print()
+                print(_format_early_reset_warning(early, protocol, protocol.minimum_terrain_size_for_bounds()))  # type: ignore[index,arg-type]
+                break
+
+        if args_cli.eval_type == "all" and args_cli.include_summary:
+            runs = []
+            for seed in protocol.seeds:
+                data = _spawn_stage("plain", seed)
+                runs.append((seed, data["metrics"]))  # type: ignore[arg-type]
+            result["summary"] = aggregate_runs(runs)  # type: ignore[arg-type]
+            print()
+            print("[混合地形汇总]")
+            print(format_aggregate_summary(result["summary"]))  # type: ignore[arg-type]
+            if runs:
+                result["summary_metrics"] = filter_metrics(runs[0][1], args_cli.eval_type)
+    else:
+        runs = []
+        recorder_report = {}
+        early_any = None
+        for seed in protocol.seeds:
+            data = _spawn_stage("plain", seed)
+            runs.append((seed, data["metrics"]))  # type: ignore[arg-type]
+            recorder_report = data["recorder"]  # type: ignore[assignment]
+            early_any = data["early_resets"]
+        aggregate = aggregate_runs(runs)  # type: ignore[arg-type]
+        result["aggregate"] = aggregate
+        result["metrics"] = filter_metrics(runs[0][1], args_cli.eval_type)
+        result["recorder"] = recorder_report
+        result["early_resets"] = early_any
+        print()
+        print(format_metric_result(args_cli.eval_type, runs[0][1]))
+        if len(runs) > 1:
+            print()
+            print(format_aggregate_summary(aggregate))
+        if early_any and early_any["polluted"]:  # type: ignore[index]
+            print()
+            print(_format_early_reset_warning(early_any, protocol, protocol.minimum_terrain_size_for_bounds()))  # type: ignore[index,arg-type]
+
+    warnings = result.get("recorder", {}).get("warnings")  # type: ignore[union-attr]
+    if warnings:
+        print("\n[形态解析警告]")
+        for warning in warnings:  # type: ignore[union-attr]
+            print("  -", warning)
+
+    output_path = _resolve_output_path(args_cli, resume_path, args_cli.eval_type)
+    with open(output_path, "w", encoding="utf-8") as handle:
+        json.dump(result, handle, indent=2, ensure_ascii=False)
+    print(f"\n[eval_kit] 结果已保存：{output_path}")
+    if args_cli.print_json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+
+
+def _run_child(
+    args_cli: argparse.Namespace,
+    *,
+    viewer_env_index: int | None,
+    free_camera: bool,
+    livestream: int,
+    pause_s: float,
+) -> None:
+    """子进程：启动仿真，只建一个环境，跑完把结果写进 ``--stage-out``。
+
+    ``viewer_env_index`` / ``free_camera`` / ``livestream`` / ``pause_s`` 都由
+    :func:`main` 在 ``AppLauncher`` 之前算好传进来：构造 ``AppLauncher`` 会改写
+    ``headless`` 并把 ``livestream`` 从 namespace 里 pop 掉，之后再读就晚了。
+    """
+    print(f"[eval_kit][child] stage={args_cli.stage} seed={args_cli.stage_seed} 开始", flush=True)
+    import numpy as np
+
+    from isaaclab_rl.rsl_rl import RslRlOnPolicyRunnerCfg
+    from isaaclab_tasks.utils.hydra import hydra_task_config
+
+    import rl_training  # noqa: F401  触发任务注册
+    import rl_training.tasks  # noqa: F401
+
+    from eval_kit.core.aggregation import build_upper_limit_matrix
+    from eval_kit.core.metrics import compute_metrics
+    from eval_kit.isaaclab import env_setup
+    from eval_kit.isaaclab.policy import load_policy, make_env, resolve_checkpoint
+    from eval_kit.isaaclab.recorder import build_recorder, record_rollout, summarize_early_resets
+
+    profile = get_profile(args_cli.profile)
+    # 子进程只跑一个种子：种子的覆盖由父进程通过 --stage-seed 传进来
+    protocol = replace(_apply_overrides(get_protocol(args_cli.protocol), args_cli), seeds=(args_cli.stage_seed,))
+    spec = get_terrain_spec(protocol.terrain_spec)
+    if args_cli.robot_mass is not None:
+        profile = replace(profile, mass_kg=args_cli.robot_mass)
+
+    @hydra_task_config(args_cli.task, args_cli.agent)
+    def _child(env_cfg, agent_cfg: RslRlOnPolicyRunnerCfg):
+        experiment_name = args_cli.experiment_name or agent_cfg.experiment_name
+        log_root_path = os.path.abspath(os.path.join("logs", "rsl_rl", experiment_name))
+        resume_path = resolve_checkpoint(log_root_path, args_cli.load_run, args_cli.checkpoint)
+
+        cfg = copy.deepcopy(env_cfg)
+        if args_cli.stage == "upper":
+            terrain_types = _terrain_types_for(args_cli, spec)
+            env_setup.configure_upper_limit_env(
+                cfg, spec, protocol, terrain_types,
+                seed=args_cli.stage_seed, device=args_cli.device,
+                viewer_env_index=viewer_env_index, free_camera=free_camera,
+            )
+        else:
+            terrain_types = []
+            env_setup.configure_eval_env(
+                cfg, spec, protocol, num_envs=args_cli.num_envs,
+                device=args_cli.device, seed=args_cli.stage_seed,
+                viewer_env_index=viewer_env_index, free_camera=free_camera,
+            )
+
+        if viewer_env_index is not None:
+            num_envs = int(cfg.scene.num_envs)
+            if not (0 <= viewer_env_index < num_envs):
+                raise SystemExit(
+                    f"--view-env {viewer_env_index} 超出范围：本次共 {num_envs} 个环境（编号 0~{num_envs - 1}）。"
+                )
+            if terrain_types:
+                per_type = protocol.num_levels * protocol.envs_per_cell
+                type_index, remainder = divmod(viewer_env_index, per_type)
+                level_index = remainder // protocol.envs_per_cell
+                print(
+                    f"[eval_kit] 视口跟随 env {viewer_env_index} → 地形 {terrain_types[type_index]}"
+                    f"，难度等级 L{level_index + 1}（共 {num_envs} 个环境）",
+                    flush=True,
+                )
+            else:
+                print(f"[eval_kit] 视口跟随 env {viewer_env_index}（共 {num_envs} 个环境）", flush=True)
+
+        env = make_env(args_cli.task, cfg, agent_cfg.clip_actions)
+        recorder = build_recorder(env, profile)
+        try:
+            # 策略跟仿真放在同一张卡上：默认的 --device 也会覆盖 agent 配置里的
+            # device，否则在别人占着 cuda:0 的机器上会莫名其妙 OOM。
+            policy = load_policy(env, agent_cfg, resume_path, device=args_cli.device)
+            if args_cli.record_frames:
+                policy = _with_frame_capture(policy, args_cli.record_frames, int(args_cli.record_every))
+            if args_cli.stage == "upper":
+                level_ids, type_ids = env_setup.assign_upper_limit_origins(env, spec, protocol, terrain_types)
+            if free_camera:
+                _set_free_camera_overview(env)
+            if pause_s > 0:
+                _hold_before_rollout(
+                    env.unwrapped.sim,
+                    pause_s,
+                    start_height_note=(
+                        f"（本次起始高度被抬高了 {protocol.start_z_offset_m:.2f} m，"
+                        f"所以是离地悬着的）"
+                        if protocol.start_z_offset_m > 0
+                        else ""
+                    ),
+                )
+            trace = record_rollout(env, policy, protocol, profile, recorder)
+            if args_cli.record_frames:
+                _report_recorded_frames(args_cli.record_frames, int(args_cli.record_every), protocol.dt)
+            metrics = compute_metrics(
+                trace, profile, protocol, include_per_env=(args_cli.stage == "upper")
+            )
+            early = summarize_early_resets(trace)
+        finally:
+            env.close()
+
+        if args_cli.save_trace:
+            os.makedirs(args_cli.save_trace, exist_ok=True)
+            tag = "upper" if args_cli.stage == "upper" else args_cli.eval_type
+            trace.save(os.path.join(args_cli.save_trace, f"{tag}_seed{args_cli.stage_seed}.npz"))
+
+        payload: dict[str, object] = {
+            "seed": args_cli.stage_seed,
+            "metrics": metrics,
+            "early_resets": early,
+            "recorder": _recorder_report(recorder),
+        }
+        if args_cli.stage == "upper":
+            payload["per_env"] = metrics.pop("_per_env")
+            payload["level_ids"] = level_ids.cpu().numpy().tolist()
+            payload["type_ids"] = type_ids.cpu().numpy().tolist()
+        with open(args_cli.stage_out, "w", encoding="utf-8") as handle:
+            json.dump(payload, handle)
+
+    _child()
+    print("[eval_kit][child] 完成", flush=True)
+
+
+def main() -> None:
+    parser = _build_parser()
+    AppLauncher.add_app_launcher_args(parser)
+    args_cli, hydra_args = parser.parse_known_args()
+
+    # 父进程只做编排：它不建环境，因此也不需要启动仿真。
+    if args_cli.stage == "":
+        if _livestream_value(args_cli) > 0 and not args_cli.headless:
+            print(
+                "[eval_kit] 注意：--livestream 会自动进入 headless 模式（推流由服务端渲染），"
+                "不需要也不应该再传 --headless。",
+                flush=True,
+            )
+        _run_parent(args_cli)
+        return
+
+    # 这三个值必须在 AppLauncher 之前定下来（见 _viewer_env_index 的说明）。
+    livestream = _livestream_value(args_cli)
+    viewer_env_index = _viewer_env_index(args_cli)
+    free_camera = bool(getattr(args_cli, "free_camera", False)) and _display_enabled(args_cli)
+    pause_s = args_cli.pause_s if args_cli.pause_s is not None else (30.0 if livestream > 0 else 0.0)
+    if args_cli.free_camera and not free_camera:
+        print(
+            "[eval_kit] 提示：--free-camera 只在有画面时（本地 GUI 或 --livestream）有效，"
+            "本次是 headless 且未推流，忽略该参数。",
+            flush=True,
+        )
+    if livestream > 0:
+        _allow_client_resize(args_cli)
+
+    # 子进程：启动仿真，跑一个环境，然后退出。
+    sys.argv = [sys.argv[0]] + hydra_args
+    simulation_app = AppLauncher(args_cli).app
+    try:
+        if livestream > 0:
+            _print_livestream_info(livestream)
+        _run_child(
+            args_cli,
+            viewer_env_index=viewer_env_index,
+            free_camera=free_camera,
+            livestream=livestream,
+            pause_s=pause_s,
+        )
+    except BaseException:
+        # 关键：不能走 simulation_app.close()。它会调用 os._exit(0)，把「子进程失败」
+        # 伪装成「成功」，并且丢掉还没 flush 的 stderr —— 于是父进程只看到退出码 0
+        # 和一个空的 JSON，真实报错（例如资产配置里的 SyntaxError）完全看不到。
+        # 先把报错刷出去，再以非零码强制退出，父进程才会把真实错误报上来。
+        import traceback
+
+        traceback.print_exc()
+        for stream in (sys.stdout, sys.stderr):
+            stream.flush()
+        os._exit(1)
+    for stream in (sys.stdout, sys.stderr):
+        stream.flush()
+    simulation_app.close()
+
+
+
+if __name__ == "__main__":
+    main()