diff --git a/examples-guards/areal/examples_manifest.yaml b/examples-guards/areal/examples_manifest.yaml index 895f3c480..5492c0b0b 100644 --- a/examples-guards/areal/examples_manifest.yaml +++ b/examples-guards/areal/examples_manifest.yaml @@ -2,11 +2,8 @@ version: 1 scan: root: examples include_extensions: ['.py', '.sh'] - # files-only 模式(docs/examples-guard-engine.md §2.7):scan 只保留 - # root + include_extensions 两个键;实现模块、上游自测等一律进 - # unsupported 段(带分组注释与理由)。 - # 对账口径:release 树(当前 v2.1.0)114 个 .py/.sh = - # supported(18) + unsupported(96)。 + # 当前 v2.1.1,扫描共 161 个 + # 支持用例(4)+ 可看护暂不支持用例(34)+ 不可看护对象(123) # 看护 release 线(workflow target_ref 留空,由引擎按 latest release # 解析)。supported 配方用专用镜像 ghcr.io/hwvanici/areal_npu:v1.0.5-a2 @@ -19,134 +16,35 @@ scan: # - scheduler.type=null 的 yaml(math 系)必须补 scheduler.type=local。 # - norm:reward=group + adv=batch 实测可过;reward/adv 为 batch 级的需加 # mean_level=null(HCCL 不支持 float64 allreduce)。 -# - setup_timeout=600 兜底数据集在线冷下载(HF_ENDPOINT=huggingface.co)。 +# - setup_timeout=600 兜底数据集在线冷下载(HF_ENDPOINT=hf-mirror.com)。 # - rollout.setup_timeout 默认仅 300s,vLLM-Ascend 多模态 server 预热(mm # processor warmup + tokenizer 深拷贝)在慢机上会超,RL 条目统一 # +rollout.setup_timeout=1800 兜底。 supported: - # CLEVR 计数 GRPO(复用 areal-vlm-grpo profile;重数据:10.4GB 下载 + - # 7 万图单进程预处理,故 setup_timeout 3600 / timeout 180) - # 已知问题:offload+disk 的存盘聚合走 CPU gloo,会超上游 120s 硬编码等待 - # (remote_inf_engine.py);1.5B 条目已去 offload 规避,3B 待内存验证。 - - path: examples/vlm/clevr_count_70k_grpo.py - profile: areal-vlm-grpo + # SWE 轨迹 SFT(纯本地,无 rollout/沙箱;数据用 fixture 的 pair-format JSONL, + # 路径含 swe token 自动路由进 SWE 管线;配置复用 gsm8k_sft.yaml + swe 覆盖) + - path: examples/swe/train_sft.py + profile: areal-swe-sft runner: linux-aarch64-a2-2 npu_devices: '0,1' image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 overlay_args: - - --config examples/vlm/clevr_count_70k_grpo.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - +rollout.setup_timeout=1800 - - actor.backend=fsdp:d2p1t1 - - +actor.fsdp.offload_params=true - - +actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.35 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=3600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - - vllm.max_model_len=8576 - timeout_minutes: 300 - # tau2-bench 客服多轮 RL(保留真实 UserSimulator,solo_mode=false):用户侧由 - # run_example 在另一张卡拉本地 vLLM(同一份 Qwen3-1.7B 权重、多 served-model-name - # 别名吸收 econfig.user_llm 的 litellm 命名),econfig.user_llm_base_url 指过去; - # agent 侧照常走 AReaL proxy。max_steps/new_tokens/n_samples/batch_size 压到 - # 冒烟量级(单集远低于入口硬编码的 600s episode 超时)。 - - path: examples/tau2/train.py - profile: areal-tau2-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/tau2/config_1.7b_airline.yaml - - cluster.n_gpus_per_node=1 - - rollout.backend=vllm:d1p1t1 - - actor.backend=fsdp:d1p1t1 - - +actor.fsdp.offload_params=true - - +actor.weight_update_mode=disk - - actor.enable_tree_training=false - # mb 容量必须 ≥ 单条最长轨迹(tau2 airline 轨迹 ~4978 token,1024 时 - # ffd_allocate 直接 RuntimeError);24576+pad_to_maximum=true 实测 OOM - # (4978 垫到 24576 的 5 倍 padding)——8192 + 不垫的组合。 - - actor.pad_to_maximum=false - - actor.mb_spec.max_tokens_per_mb=8192 - - ref.mb_spec.max_tokens_per_mb=8192 - - actor.path=${AREAL_MODEL_PATH} - - econfig.max_steps=2 - - econfig.user_llm_base_url=http://127.0.0.1:18888/v1/ - - econfig.user_llm_args.max_completion_tokens=64 - - gconfig.n_samples=1 - - gconfig.max_new_tokens=64 - - gconfig.max_tokens=8192 - - train_dataset.batch_size=1 - - valid_dataset.batch_size=1 - - stats_logger.wandb.mode=disabled - - vllm.gpu_memory_utilization=0.3 - - vllm.max_model_len=8192 - - +rollout.setup_timeout=1800 - - total_train_steps=1 - - total_train_epochs=1 - timeout_minutes: 150 - # Qwen2.5-VL-3B Geometry3K GRPO(双卡) - - path: examples/vlm/geometry3k_grpo.py - profile: areal-vlm-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/vlm_npu/qwen2_5_vl_3b_geometry3k_grpo.yaml + - --config examples/math/gsm8k_sft.yaml - scheduler.type=local - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - # mm 预热会超默认 300s 健康等待(mm server 预热见头部通用配方) - - +rollout.setup_timeout=1800 - actor.backend=fsdp:d2p1t1 - - +actor.fsdp.offload_params=true - - +actor.weight_update_mode=disk - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 + - experiment_name=swe-sft-smoke - train_dataset.batch_size=2 + - train_dataset.path=${FIXTURE_DIR}/swe_sft_pairs.jsonl - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 + - valid_dataset.path=${FIXTURE_DIR}/swe_sft_pairs.jsonl + - +swe.pre_split=true + - +swe.dump_samples=0 - +total_train_steps=1 - - vllm.max_model_len=8576 - timeout_minutes: 150 - # GSM8K GRPO(用上游 NPU 专用 yaml) - - path: examples/math/gsm8k_rl.py - profile: areal-math-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/math/gsm8k_grpo_npu.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - actor.backend=fsdp:d2p1t1 - - ++actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - +train_dataset.setup_timeout=600 - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - timeout_minutes: 150 + timeout_minutes: 90 # GSM8K SFT(纯 FSDP,无 rollout) - path: examples/math/gsm8k_sft.py profile: areal-math-sft @@ -165,265 +63,7 @@ supported: - +valid_dataset.setup_timeout=600 - +total_train_steps=1 timeout_minutes: 150 - # BoBa GRPO(本地 jsonl,setup 预下载到 AREAL_BOBA_DATA) - - path: examples/math/boba_grpo.py - profile: areal-math-boba - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/math/boba_grpo.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - actor.backend=fsdp:d2p1t1 - - ++actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - actor.ppo_n_minibatches=1 - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - rollout.enable_rollout_tracing=false - - perf_tracer.enabled=false - - train_dataset.path=${AREAL_BOBA_DATA} - - train_dataset.batch_size=2 - - +total_train_steps=1 - - ++actor.scheduling_spec.0.env_vars.TASK_QUEUE_ENABLE=1 - timeout_minutes: 150 - # Countdown GRPO(数据由 setup 现场生成) - - path: examples/countdown/train.py - profile: areal-countdown-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/countdown/train_config.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - +rollout.setup_timeout=1800 - - actor.backend=fsdp:d2p1t1 - - +actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - +total_train_steps=1 - timeout_minutes: 150 - # GSM8K 多轮自我纠错 GRPO(复用 areal-math-grpo profile) - - path: examples/multi_turn_math/gsm8k_rl_mt.py - profile: areal-math-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/multi_turn_math/gsm8k_grpo_mt.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - actor.backend=fsdp:d2p1t1 - - +actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - timeout_minutes: 150 - # TIR 工具集成推理 GRPO(带内工具调用 + 本地 python/calculator;torl 数据 - # 仅 GitHub 来源、job 容器不可达,parquet 以 fixture 入库) - - path: examples/tir/train_tir.py - profile: areal-tir-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/tir/tir_math_config.yaml - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - +rollout.setup_timeout=1800 - - actor.backend=fsdp:d2p1t1 - - +actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=128 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - - vllm.max_model_len=4096 - timeout_minutes: 150 - # Scaffolding RLVR(旧版 SPMD yaml:+scheduler/+backend,~ 删 legacy 键, - # python -m 启动);worker 写死 model="default" 而 vLLM 按模型路径名服务会 - # 404,故 setup 建 TARGET_ROOT/default 软链 + vllm.model=default。 - - path: examples/scaffolding/gsm8k_rlvr_scaffolding.py - profile: areal-scaffold-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/scaffolding/gsm8k_rlvr_scaffolding.yaml - - +scheduler.type=local - - cluster.n_gpus_per_node=2 - - +rollout.backend=vllm:d1p1t2 - - +rollout.setup_timeout=1800 - - +actor.backend=fsdp:d2p1t1 - - +ref.backend=fsdp:d2p1t1 - - +actor.fsdp.offload_params=true - - ~actor.behav_imp_weight_cap - - actor.path=${AREAL_MODEL_PATH} - - vllm.model=default - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - timeout_minutes: 150 - # OpenAI Agents SDK 多 agent handoff GRPO(LLM 走本地 vLLM) - - path: examples/openai_agents/train_agents.py - profile: areal-agents-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/openai_agents/config.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - +rollout.setup_timeout=1800 - - actor.backend=fsdp:d2p1t1 - - +actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - +gconfig.max_new_tokens=64 - - gconfig.max_tokens=256 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - timeout_minutes: 150 - # camel-ai ChatAgent 多轮 GRPO(GSM8K)。入口脚本内定义了 Config,必须由 - # run_example.sh 以「import 成模块再调 main()」启动;camel-ai 由 setup 另装。 - - path: examples/camel/train.py - profile: areal-camel-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/camel/config.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - actor.backend=fsdp:d2p1t1 - - +actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - vllm.gpu_memory_utilization=0.3 - - gconfig.max_new_tokens=64 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - timeout_minutes: 150 - # Agent Workflow(OpenAI SDK agent,GSM8K)。agent 用 AReaL 注入的本地 proxy - # endpoint(非外部 LLM、无需 key);复用 areal-agents-grpo。 - - path: examples/agent_workflow/train.py - profile: areal-agents-grpo - runner: linux-aarch64-a2-2 - npu_devices: '0,1' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/agent_workflow/config.yaml - - scheduler.type=local - - cluster.n_gpus_per_node=2 - - rollout.backend=vllm:d1p1t2 - - +rollout.setup_timeout=1800 - - actor.backend=fsdp:d2p1t1 - - ++actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - actor.mb_spec.max_tokens_per_mb=1024 - - ref.mb_spec.max_tokens_per_mb=1024 - - vllm.gpu_memory_utilization=0.3 - - ++gconfig.max_new_tokens=64 - - gconfig.max_tokens=256 - - gconfig.n_samples=1 - - train_dataset.batch_size=2 - - valid_dataset.batch_size=2 - - +train_dataset.setup_timeout=600 - - +valid_dataset.setup_timeout=600 - - +total_train_steps=1 - timeout_minutes: 150 - # Hermes 在线 RL(唯一覆盖 online 模式 + Agent Service + per-session key); - # _version 回退 v1(默认 v2 强制 SGLang-only 的 awex,vLLM 404)。 - - path: examples/hermes/train.py - profile: areal-hermes-grpo - runner: linux-aarch64-a2-1 - npu_devices: '0' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/hermes/config.yaml - - rollout.backend=vllm:d1p1t1 - - rollout._version=v1 - - actor.backend=fsdp:d1p1t1 - - actor._version=v1 - - ++actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - rollout.admin_api_key=sk-areal-ci - - +rollout.agent.admin_api_key=sk-areal-ci - - vllm.gpu_memory_utilization=0.25 - - vllm.max_model_len=16384 - - gconfig.max_new_tokens=256 - - +rollout.setup_timeout=1800 - - total_train_steps=1 - - total_train_epochs=1 - timeout_minutes: 150 - # OpenClaw 在线 RL:任意 OpenAI 兼容 agent runtime(chat-completions 协议, - # 无 runtime 依赖);驱动由 run_example.sh 直接 HTTP。 - - path: examples/openclaw/train.py - profile: areal-openclaw-grpo - runner: linux-aarch64-a2-1 - npu_devices: '0' - image: ghcr.nju.edu.cn/hwvanici/areal_npu:v1.0.5-a2 - overlay_args: - - --config examples/openclaw/config.yaml - - rollout.backend=vllm:d1p1t1 - - actor.backend=fsdp:d1p1t1 - - ++actor.weight_update_mode=disk - - actor.path=${AREAL_MODEL_PATH} - - train_dataset.batch_size=1 - - actor.reward_norm=null - - actor.adv_norm=null - - vllm.gpu_memory_utilization=0.2 - - vllm.max_model_len=16384 - - gconfig.max_new_tokens=256 - - +rollout.setup_timeout=1800 - - total_train_steps=1 - - total_train_epochs=1 - timeout_minutes: 150 # 在线推理服务(rollout-only):online_rollout.py 起 gateway 后等外部流量; # run_example.sh 用直接 HTTP 造流量(start_session→chat/completions→set_reward # →refresh),收满 batch_size 轨迹即打印 Rollout complete 退出。 @@ -471,28 +111,43 @@ supported: - ++rollout.setup_timeout=1800 timeout_minutes: 150 unsupported: - # ===== 上游源码 bug(改上游源码才能通,无 overlay 可绕)======================= - # 以下 5 条都是上游脚本 / 引擎自身的 bug,实测必崩: - # 1) clevr SFT:loader 读错 processor 键名——clevr_count_70k.py:110 用旧键 - # token_type_ids 取 mm_token_type_ids(transformers 5.x 下恒 None),SFT - # collate 阶段 torch.tensor(None) 崩。(geometry3k_grpo 读同一错键无碍:GRPO - # 不 collate 数据集样本。) - # 2) hhrlhf RW:引擎在打包前就读 x["cu_seqlens"](rw_engine.py:52→21),而 collate - # 只产 input_ids/attention_mask(rw_trainer.py:61-73)→ KeyError。 - # 3) hhrlhf DPO:loss 聚合硬编码 fp64(functional.py:829-835),Ascend 不支持。 - # 4) virl39k GRPO:reward_fn 以函数对象传入,过 RPC 时 JSON 序列化失败(正常条目 - # 传"点路径"字符串);同 bug 的 vlm_npu/geometry3k_grpo.py 见下方工具段。 - # 5) agent_service:入口硬编码 LocalScheduler(experiment_name=..., trial_name=..., - # gpu_devices=[]),不传 fileroot 也不传 log_dir,local.py:158 断言必炸;CLI 仅 - # --admin-api-key、LocalScheduler 无环境变量兜底,无 overlay 出口(README 的 - # Quick Start 与 Programmatic 片段同一写法,同样必炸)。sidecar 方案本身已实测 - # 可行(本地 online_rollout 网关正常就绪),配方沉淀在 run_example.sh 的 - # agent-service 分支与 setup §1g,上游修复后可直接复用。 + # ===== 上游源码 bug(实测必崩,改上游源码才能通,无 overlay 可绕)============= + # loader 读错 processor 键名:clevr_count_70k.py:111 用旧键 token_type_ids 取 + # mm_token_type_ids(transformers 5.x 下恒 None),SFT collate 阶段 + # torch.tensor(None) 崩(geometry3k_grpo 读同一错键无碍:GRPO 不 collate 数据集样本) - examples/vlm/clevr_count_70k_sft.py + # 引擎在打包前就读 x["cu_seqlens"](rw_engine.py:22/50/126),而 collate 只产 + # input_ids/attention_mask(rw_trainer.py:55)→ KeyError - examples/alignment/hhrlhf_rw.py + # loss 聚合硬编码 fp64(functional.py:840-846,dpo_pair_logratios;docstring 载明 + # 有意设计,需上游参数化),Ascend 不支持 - examples/alignment/hhrlhf_dpo.py + # reward_fn 以函数对象传入,过 RPC 时 JSON 序列化失败(正常条目传"点路径"字符串) - examples/vlm_npu/virl39k_grpo.py + # 入口硬编码 LocalScheduler(experiment_name=..., trial_name=..., gpu_devices=[]), + # 不传 fileroot 也不传 log_dir,local.py:170 断言必炸;CLI 仅 --admin-api-key、 + # LocalScheduler 无环境变量兜底,无 overlay 出口(README 的 Quick Start 与 + # Programmatic 片段同一写法,同样必炸)。sidecar 方案本身已实测可行(本地 + # online_rollout 网关正常就绪),配方沉淀在 run_example.sh 的 agent-service 分支 + # 与 setup §1g,上游修复后可直接复用 - examples/agent_service/run_agent_service.py + # --- 上游 v2.1.1 回归:stats_tracker 改 fp64,NPU HCCL 不支持 at::kDouble --- + # 11a1907e(#1736) 把统计累加/reduce 改成 fp64;训练能跑完、最后 export_stats 崩 + # (2026-11-05 实测)。无配置/环境变量出口;钉回 v2.1.0 或等上游修复后迁回 + - examples/vlm/clevr_count_70k_grpo.py + - examples/tau2/train.py + - examples/vlm/geometry3k_grpo.py + - examples/math/gsm8k_rl.py + - examples/math/boba_grpo.py + - examples/countdown/train.py + - examples/multi_turn_math/gsm8k_rl_mt.py + - examples/tir/train_tir.py + - examples/scaffolding/gsm8k_rlvr_scaffolding.py + - examples/openai_agents/train_agents.py + - examples/camel/train.py + - examples/agent_workflow/train.py + - examples/hermes/train.py + - examples/openclaw/train.py # ===== 上游 demo 设计缺陷(时序竞态,双向不稳定;2026-10-16 实测定位)========= # hitl demo(zeroclaw 第三方 CLI 兼容性验证,价值仍在):admin key 被 @@ -528,10 +183,31 @@ unsupported: # 沙箱必须 AEnvironment 集群(k8s+Helm),CI 无集群;自建路线实测锁死(容器无 # CAP_SYS_ADMIN、mount 被拒,起不了 DinD/k8s)。回归:单容器自托管或注入 AENV_SYSTEM_URL - examples/swe/train_swe_rl.py + # Arena RL:需 Arena 在线 Stream 服务与 slurm/sbatch 集群提交 + - examples/swe/qwen38_flash_next/train_rl.py + - examples/swe/rl/qwen38_27b/qwen27_train_rl.py + # SGLang rollout 硬依赖(脚本断言),NPU 不支持 + - examples/math/gsm8k_score_centering.py + # SGLang rollout + Megatron/AWEX 栈(8 卡),NPU 不支持 + - examples/cpu_staged_offload/dapo-math_rl_cpu_staged.py + # SGLang rollout + Megatron/AWEX 栈(8 卡),NPU 不支持 + - examples/mopd/gsm8k_qwen3_14b_to_0_6b.py + # SGLang rollout + Megatron + R3 router replay(SGLang-only 特性),NPU 不支持 + - examples/r3/gsm8k_multiturn.py + # 同 r3 配置族(sglang rollout + megatron actor + Moonlight,8 卡),NPU 不支持 + - examples/r3/gsm8k_rlvr.py + # 多模态 agent 轨迹仅支持 SGLang rollout(client.py 运行时校验 vLLM 拒绝),NPU 不支持 + - examples/vlm/geometry3k_agent_grpo.py + # Daytona 云沙箱评分示例,需 DAYTONA_API_KEY(见 README),CI 无外部服务 + - examples/sandbox_daytona/reward_example.py + + # ===== 可看护但重复(与已看护用例同链路,不单列省 CI)===== + # 副本现带 virl39k 同款 RPC bug(见上),修好后仍与正主重复 + - examples/vlm_npu/geometry3k_grpo.py # examples/vlm/geometry3k_grpo.py + - # ===== 工具套件(非独立入口;被上面的入口 import 的实现 / 测试 / 包标识 / helper / CLI)===== - # scan 到但都不是可独立运行的示例,只被入口间接调用: - # -- 上游自测(scaffolding/tests、tir/tests)与 pytest 基础设施 + # ===== 不可看护的用例===== + # 上游自测与 pytest 基础设施(scaffolding/tests、tir/tests),非可守卫 example。 - examples/scaffolding/tests/test_controllers.py - examples/scaffolding/tests/test_scaffolding_llm_integration.py - examples/scaffolding/tests/test_self_contained.py @@ -544,7 +220,7 @@ unsupported: - examples/tir/tests/test_tir_workflow_async.py - examples/tir/tests/test_tool_manager_async.py - examples/tir/tests/test_tool_registry_lazy.py - # -- 包标识文件(__init__.py) + # 包标识文件(__init__.py)。 - examples/hermes/__init__.py - examples/scaffolding/__init__.py - examples/scaffolding/core/__init__.py @@ -553,7 +229,13 @@ unsupported: - examples/terminal_bench/agent/__init__.py - examples/terminal_bench/workflow/__init__.py - examples/tir/tools/__init__.py - # -- scaffolding 框架实现(run_example.sh 以 python -m 跑 scaffolding/*.py 时 import) + - examples/__init__.py + - examples/cpu_staged_offload/__init__.py + - examples/swe/__init__.py + # helper / 实现模块:被同目录 example import 或经 config 引用,非入口(scaffolding / tir / + # terminal_bench / search_agent 的框架实现,各 example 的 config / utils / reward, + # data_engineering / prm / tau2 的配套模块,swe(含 Arena 工作流与 worker)/ hermes / + # openclaw / agent_service 的支撑模块)。 - examples/scaffolding/_compat.py - examples/scaffolding/controllers.py - examples/scaffolding/core/controller.py @@ -570,7 +252,6 @@ unsupported: - examples/scaffolding/task.py - examples/scaffolding/worker.py - examples/scaffolding/workflow.py - # -- tir 工具链实现(train_tir.py 调用) - examples/tir/prompts.py - examples/tir/tir_workflow.py - examples/tir/tool_manager.py @@ -578,64 +259,82 @@ unsupported: - examples/tir/tools/calculator_tool.py - examples/tir/tools/daytona_python_tool.py - examples/tir/tools/python_tool.py - # -- terminal_bench 实现(train.py 调用) - examples/terminal_bench/agent/camel_terminal_agent.py - examples/terminal_bench/agent/chat_agent_trace.py - examples/terminal_bench/agent/prompts.py - examples/terminal_bench/agent_rl_config.py - examples/terminal_bench/workflow/camel_rlvr_workflow.py - examples/terminal_bench/workflow/pre_build_tasks_utils.py - # -- search_agent/tongyi_deepresearch 实现(entry 调用) - examples/search_agent/tongyi_deepresearch/config.py - examples/search_agent/tongyi_deepresearch/prompt.py - examples/search_agent/tongyi_deepresearch/react_agent.py - examples/search_agent/tongyi_deepresearch/tool_search.py - examples/search_agent/tongyi_deepresearch/tool_visit.py - # -- 各 example 的单文件 config / utils / reward helper - examples/agent_workflow/configs.py - examples/countdown/reward_score.py - examples/multi_turn_math/config.py - examples/openai_agents/config.py - examples/tau2/utils.py - # -- profile rank helper(run_qwen3_30b_a3b_sft_profile.sh 内嵌调用) - - examples/profile/profile_rank_utils.py - # -- SWE env/train config dataclass(train_swe_rl.py import) - examples/swe/utils.py - # -- SWE rollout 过滤函数(按组内 reward 方差过滤全对/全错样本) - examples/swe/filter_function.py - # -- 其它被入口 import 的实现 / 环境 / demo + - examples/swe/agent.py + - examples/swe/prefix_matchers.py + - examples/swe/preprocessors.py - examples/agent_service/agent.py - examples/docs/debug/cmp_rollout.py - - examples/sandbox_daytona/reward_example.py - examples/tau2/agent.py - # -- openclaw 在线链路的驱动与工具(run_example 已改为直接 HTTP,这些脚本不再调用) - examples/openclaw/_fmt.py - examples/openclaw/demo_lifecycle.py - examples/openclaw/set_reward.py - examples/openclaw/start_session.py - # -- Hermes 的 AgentRunnable 实现(Agent Service 进程内每 session 加载) - examples/hermes/hermes.py - # -- SWE 的 agent workflow 实现(train_swe_rl.py 调用) - - examples/swe/agent.py - # -- SWE InteractionCache 前缀匹配器 - - examples/swe/prefix_matchers.py - # -- SWE 消息预处理器 - - examples/swe/preprocessors.py - # -- hermes 会话工具(纯 HTTP;run_example 已改为直接 HTTP,hermes_loop 不再调用) - examples/hermes/hermes_loop.py - examples/hermes/set_reward.py - examples/hermes/start_session.py - # -- SGLang 工具链补丁(环境修补工具,非入口) - - examples/swe/apply_sgl_radix_cache_patch.py - # -- profile 性能剖析三件套(CUDA 专属工具链,NPU 无对应产物) + - examples/data_engineering/tir/data_calibration.py + - examples/data_engineering/tir/data_recipe.py + - examples/prm/scorers.py + - examples/tau2/message_compat.py + - examples/swe/arena_agent.py + - examples/swe/arena_client.py + - examples/swe/arena_config.py + - examples/swe/arena_dataset_snapshot.py + - examples/swe/arena_reward_breakdown.py + - examples/swe/arena_reward_mapping.py + - examples/swe/arena_types.py + - examples/swe/config.py + - examples/swe/reward_transforms.py + - examples/swe/qwen38_flash_next/actor_worker.py + - examples/swe/qwen38_flash_next/gdn_cp_compat.py + - examples/swe/qwen38_flash_next/grad_norm_guard.py + - examples/swe/qwen38_flash_next/ple_chunked.py + - examples/swe/qwen38_flash_next/proxy.py + - examples/swe/qwen38_flash_next/template_defaults.py + - examples/swe/rl/qwen38_27b/configure_sglang_logging.py + - examples/swe/rl/qwen38_27b/qwen27_actor_worker.py + # 工具 / CLI:数据生成、评估器、profile 剖析工具链(CUDA 专属)、SGLang 补丁与 Arena 提交/校验工具,非入口。 + - examples/countdown/countdown.py + - examples/math/gsm8k_eval.py + - examples/profile/profile_rank_utils.py - examples/profile/postprocess_profile.py - examples/profile/run_qwen3_30b_a3b_sft_profile.sh - examples/profile/train_sft_profile.py - # -- 特殊 CLI(countdown 数据生成 / gsm8k_eval 评估器,非入口) - - examples/countdown/countdown.py - - examples/math/gsm8k_eval.py - # -- vlm_npu 包装层 / 坏副本(geometry3k_grpo.py 同 virl39k 的 RPC bug;.sh 为包装) - - examples/vlm_npu/geometry3k_grpo.py + - examples/swe/apply_sgl_radix_cache_patch.py + - examples/swe/arena_preflight.py + - examples/swe/arena_stream_config_projection.py + - examples/swe/patch_sglang_qwen3_reasoning.py + - examples/swe/rl/qwen38_27b/source_snapshot.py + - examples/swe/rl/qwen38_27b/validate_recover_preflight.py + # .sh 脚本(vlm_npu 包装、swe slurm/sbatch 提交)。 - examples/vlm_npu/qwen2_5_vl_3b_geometry3k_grpo.sh - examples/vlm_npu/qwen2_5_vl_3b_virl39k_grpo_multinode.sh - examples/vlm_npu/qwen3_vl_2b_geometry3k_grpo.sh + - examples/swe/qwen38_flash_next/rl_controller.sh + - examples/swe/qwen38_flash_next/sbatch_sft_qwen38_flash_next.sh + - examples/swe/qwen38_flash_next/sft_worker.sh + - examples/swe/qwen38_flash_next/submit_rl.sh + - examples/swe/qwen38_flash_next/verify_bridge.sh + - examples/swe/rl/qwen38_27b/runtime.sh + - examples/swe/rl/qwen38_27b/submit.sh + - examples/swe/submit_arena.sh + - examples/swe/submit_arena_flash_v3.sh diff --git a/examples-guards/areal/fixtures/swe_sft_pairs.jsonl b/examples-guards/areal/fixtures/swe_sft_pairs.jsonl new file mode 100644 index 000000000..37fb85e3e --- /dev/null +++ b/examples-guards/areal/fixtures/swe_sft_pairs.jsonl @@ -0,0 +1,4 @@ +{"messages": [{"role": "user", "content": "utils.py 的 parse_date 对 '2024-02-30' 抛 ValueError,请修复为把非法日期收敛到当月最后一天。"}, {"role": "assistant", "content": "问题在构造时用静态 31 天上限校验日期。把日期收敛到当月实际天数即可:\n\n```python\nimport calendar\n\ndef parse_date(s):\n year, month, day = map(int, s.split('-'))\n day = min(day, calendar.monthrange(year, month)[1])\n return datetime.date(year, month, day)\n```"}]} +{"messages": [{"role": "user", "content": "tests/test_cache.py 挂了:clear() 之后 LRU 索引里还有残留 key,怎么修?"}, {"role": "assistant", "content": "clear() 只清了数据字典,没有清有序索引,导致查询命中幽灵条目。两个结构都要重置:\n\n```python\ndef clear(self):\n self._data.clear()\n self._index.clear()\n```"}]} +{"messages": [{"role": "user", "content": "api/client.py 的重试逻辑在超时后没有退避,直接打爆了下游。"}, {"role": "assistant", "content": "重试之间需要指数退避。最小改法:\n\n```python\nfor attempt in range(self.max_retries):\n try:\n return self.request(url)\n except TimeoutError:\n time.sleep(2 ** attempt)\n```"}, {"role": "user", "content": "这样最后一次失败也会 sleep,能不能避免?"}, {"role": "assistant", "content": "把 sleep 放在非最后一次重试之前即可:\n\n```python\nfor attempt in range(self.max_retries):\n try:\n return self.request(url)\n except TimeoutError:\n if attempt < self.max_retries - 1:\n time.sleep(2 ** attempt)\n```"}]} +{"messages": [{"role": "user", "content": "models/user.py 的 to_dict() 递归序列化遇到 datetime 就崩。"}, {"role": "assistant", "content": "给序列化加一个类型分派,datetime 转 ISO 字符串:\n\n```python\ndef to_dict(obj):\n if isinstance(obj, datetime.datetime):\n return obj.isoformat()\n if hasattr(obj, '__dict__'):\n return {k: to_dict(v) for k, v in obj.__dict__.items()}\n return obj\n```"}, {"role": "user", "content": "测试里嵌套的 list 也会崩,怎么处理?"}, {"role": "assistant", "content": "补上 list 分支,逐项递归:\n\n```python\ndef to_dict(obj):\n if isinstance(obj, datetime.datetime):\n return obj.isoformat()\n if isinstance(obj, list):\n return [to_dict(item) for item in obj]\n if hasattr(obj, '__dict__'):\n return {k: to_dict(v) for k, v in obj.__dict__.items()}\n return obj\n```"}]} diff --git a/examples-guards/areal/scripts/run_example.sh b/examples-guards/areal/scripts/run_example.sh index 803c73ee4..2c6d3ba87 100644 --- a/examples-guards/areal/scripts/run_example.sh +++ b/examples-guards/areal/scripts/run_example.sh @@ -49,9 +49,6 @@ cd "$TARGET_ROOT" # (e.g. boba_grpo.py's reward_fn "examples.math.boba_grpo.boba_reward_fn"); # the example dir covers sibling modules imported by the script itself. export PYTHONPATH="$TARGET_ROOT:$(dirname "$LAUNCH_PATH")${PYTHONPATH:+:$PYTHONPATH}" -# examples/scaffolding/*.py use package-relative imports (from ._compat -# import ...), so they must run as modules, not as plain scripts. -# # Forked runtime logs (data-worker / data-router / data-gateway under # /logs/) are NOT part of this job's stdout, so a crash there is # otherwise invisible; dump them when the example fails. @@ -208,288 +205,6 @@ if [[ "$LAUNCH_PATH" == *.py ]]; then fi case "$EXAMPLE_REL" in - examples/scaffolding/*.py) - MODULE="${EXAMPLE_REL%.py}" - MODULE="${MODULE//\//.}" - "$PYTHON" -m "$MODULE" "${EXTRA_ARGS[@]}" || run_rc=$? - ;; - examples/hermes/train.py) - # Online RL: train.py blocks waiting for an externally-driven session - # lifecycle, so a plain foreground run would hang until timeout. - # Orchestrate one full loop instead: background trainer (starts the - # vLLM rollout engine + proxy gateway) -> Agent Service (HermesAgent) - # -> start_session over HTTP (mint session key; see step 3 for why the - # CLI is bypassed) -> one piped conversation via hermes_loop (EOF exits - # cleanly) -> set_reward -> start_session refresh (ends the session, - # exports the trajectory) -> the trainer's _OnlineAgent unblocks, trains - # its single step and exits. - HERMES_ADMIN_KEY="areal-agent-ci" - TRAIN_LOG="$PWD/hermes_train.log" - AGENT_LOG="$PWD/hermes_agent.log" - - hermes_cleanup() { - areal agent stop >/dev/null 2>&1 || true - if [[ -n "${TRAINER_PID:-}" ]] && kill -0 "$TRAINER_PID" 2>/dev/null; then - kill "$TRAINER_PID" 2>/dev/null || true - fi - } - trap hermes_cleanup EXIT - - # 1) Trainer in the background; its log is the source of truth for - # the proxy/inference gateway address (rl_trainer logs - # "Proxy gateway available at http://host:port" once online mode - # is up - engine load takes minutes). - "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" >"$TRAIN_LOG" 2>&1 & - TRAINER_PID=$! - - GW=$(wait_gateway "$TRAIN_LOG" "$TRAINER_PID" trainer) || exit 1 - echo "inference gateway: $GW" - - # 2) Agent Service with the Hermes agent (run blocks until the stack - # is healthy, then exits leaving the service up). - # hermes-agent enables 30+ toolsets by default; their definitions ride - # EVERY /v1/responses request and overflow vllm's context, making the - # agent death-loop its own compressor. HermesAgent reads - # HERMES_DISABLED_TOOLSETS in __init__ (hermes.py:118) - i.e. in the - # Agent Service worker process - so it must be EXPORTED here, BEFORE - # `areal agent run`, for the workers to inherit it. Keep only "clarify". - export HERMES_DISABLED_TOOLSETS="web,search,x_search,vision,video,image_gen,video_gen,computer_use,terminal,skills,browser,cronjob,file,tts,todo,memory,context_engine,session_search,project,code_execution,delegation,homeassistant,kanban,discord,discord_admin,yuanbao,feishu_doc,feishu_drive,spotify,debugging,safe,coding" - if ! areal agent run --agent examples.hermes.hermes.HermesAgent \ - --admin-api-key "$HERMES_ADMIN_KEY" --force >"$AGENT_LOG" 2>&1; then - echo "areal agent run failed; tail of $AGENT_LOG:" - tail -n 120 "$AGENT_LOG" - exit 1 - fi - AGENT_GW=$(grep -oE 'gateway=[A-Za-z0-9._:/-]+' "$AGENT_LOG" | tail -1 | cut -d= -f2 || true) - if [[ -z "$AGENT_GW" ]]; then - AGENT_GW=$(areal agent status 2>/dev/null | grep -oE 'https?://[A-Za-z0-9._:-]+' | tail -1 || true) - fi - if [[ -z "$AGENT_GW" ]]; then - echo "agent gateway not found; tail of $AGENT_LOG:" - tail -n 120 "$AGENT_LOG" - exit 1 - fi - echo "agent gateway: $AGENT_GW" - - # 3) Mint the session key on the inference gateway. Talk HTTP directly - # instead of examples/hermes/start_session.py: that CLI hardcodes the - # V2 gateway contract (HTTP 201 + {"sessions":[{"session_api_key":..}]}) - # while the v1 proxy gateway returns 200 + flat {"api_key","session_id"} - # (proxy_gateway.py NEW path returns the worker response as-is) - the - # CLI dies on a perfectly successful session, the retry loop then - # spawns zombie sessions until all 3 workers' capacity is exhausted - # and everything 429s. 429 ("no capacity") is retried: workers only - # register in the readiness queue after the trainer is fully up. - SESS_KEY="" - for attempt in $(seq 1 24); do - BODY=$(http_post "$GW" rl/start_session sk-areal-ci '{"task_id":"ci-hermes-smoke"}' 180) - SESS_KEY=$(printf '%s' "$BODY" | json_pick '["api_key"]') - [[ -n "$SESS_KEY" ]] && break - echo "[session attempt $attempt] no session yet (capacity/registering)" - sleep 5 - done - if [[ -z "$SESS_KEY" ]]; then - echo "start_session never returned a session key" - tail -n 40 "$TRAIN_LOG" - exit 1 - fi - echo "session: $SESS_KEY" - - # 4) Drive ONE agent turn directly over HTTP, bypassing hermes_loop.py: - # that CLI hardcodes its httpx client timeout to 120s (:47), which the - # ~11.5K-token Hermes round-trip exceeds on this NPU - it dies with - # "Read timed out" while the server is still working. Sending the same - # /v1/responses POST ourselves with a 600s client timeout avoids the - # hardcoded limit (no source patch). inf-* route the agent's LLM calls - # through the inference gateway under the session key (self-evolution - # capture); model id "default" is InferenceDataProxy's served name. - agent_turn() { - "$PYTHON" - "$AGENT_GW" "$HERMES_ADMIN_KEY" "$1" "$GW" "$SESS_KEY" <<'PY' -import json -import sys -import urllib.error -import urllib.request - -base, key, content, inf_base, sess_key = sys.argv[1:6] -payload = { - "input": [{"type": "message", "content": content}], - "model": "hermes-agent", - "user": "hermes-ci", - "inf_base_url": inf_base, - "inf_model": "default", - "session_api_key": sess_key, -} -req = urllib.request.Request( - f"{base.rstrip('/')}/v1/responses", - data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}, - method="POST", -) -try: - with urllib.request.urlopen(req, timeout=600) as resp: - data = json.loads(resp.read().decode()) -except urllib.error.HTTPError as e: - print(f"HTTP {e.code}: {e.read().decode()[:300]}", file=sys.stderr) - sys.exit(1) -except OSError as e: - print(f"{type(e).__name__}: {e}", file=sys.stderr) - sys.exit(1) -texts = [] -for item in data.get("output", []): - if item.get("type") == "message": - for block in item.get("content", []): - if block.get("type") == "output_text": - texts.append(block["text"]) -print("".join(texts) if texts else json.dumps(data)[:300]) -PY - } - - if ! AGENT_REPLY=$(agent_turn "Say hello and introduce yourself in one sentence."); then - echo "agent turn failed; tails:" - tail -n 60 "$AGENT_LOG" - tail -n 40 "$TRAIN_LOG" - exit 1 - fi - echo "agent reply: $AGENT_REPLY" - - # 5) Reward the session, then refresh it: the refresh call (start_session - # with the existing api_key) ends the old session, waits for the - # training pipeline to cycle and starts a new one with the same key - - # the trajectory export unblocks the trainer's _OnlineAgent. Both go - # through http_post (v1 contract: 200 on success). Refresh can 429 while - # the pipeline cycles (refresh_timeout on the gateway side), so retry. - REWARD_RESP=$(http_post "$GW" rl/set_reward "$SESS_KEY" '{"reward":1.0}' 180) - if [[ -z "$REWARD_RESP" ]]; then - echo "set_reward failed; tail of $TRAIN_LOG:" - tail -n 40 "$TRAIN_LOG" - exit 1 - fi - echo "reward applied: $REWARD_RESP" - - REFRESHED="" - for attempt in $(seq 1 12); do - BODY=$(http_post "$GW" rl/start_session sk-areal-ci \ - "{\"task_id\":\"ci-hermes-smoke\",\"api_key\":\"$SESS_KEY\"}" 180) - NEW_KEY=$(printf '%s' "$BODY" | json_pick '["api_key"]') - if [[ -n "$NEW_KEY" ]]; then - REFRESHED=1 - echo "session refreshed (key: $NEW_KEY)" - break - fi - echo "[refresh attempt $attempt] pipeline still cycling" - sleep 10 - done - if [[ -z "$REFRESHED" ]]; then - echo "session refresh never succeeded; the trainer may be stuck waiting" - tail -n 40 "$TRAIN_LOG" - exit 1 - fi - - # 6) Trainer trains its single step and exits. Bounded wait: a rejected - # trajectory (empty/error-only session) leaves the trainer waiting - # for the NEXT session forever - instead of holding the runner until - # the job timeout, fail after 15 min with the trainer log dumped. - wait_bounded "$TRAINER_PID" 900 "$TRAIN_LOG" - if (( WAIT_TIMED_OUT )); then - run_rc=1 - echo "trainer still running after 900s - likely waiting for another session (rejected trajectory?); tail of $TRAIN_LOG:" - tail -n 120 "$TRAIN_LOG" - elif wait "$TRAINER_PID"; then - echo "hermes online-rl loop completed" - else - run_rc=$? - echo "trainer exited rc=$run_rc; tail of $TRAIN_LOG:" - tail -n 120 "$TRAIN_LOG" - fi - ;; - examples/openclaw/train.py) - # Online RL via the proxy gateway (same architecture as hermes): the agent - # runtime is any OpenAI-compatible client. We drive one episode DIRECTLY - # over HTTP instead of the upstream examples/openclaw/demo_lifecycle.py, - # whose /chat/completions client timeout is hardcoded to 30s (:264) - the - # NPU generation exceeds that, so it dies with "Read timed out" before - # set_reward. Doing the same calls ourselves with a long client timeout - # avoids the hardcoded limit (no source patch). - OC_ADMIN_KEY="sk-test123456" - TRAIN_LOG="$PWD/openclaw_train.log" - - openclaw_cleanup() { - if [[ -n "${TRAINER_PID:-}" ]] && kill -0 "$TRAINER_PID" 2>/dev/null; then - kill "$TRAINER_PID" 2>/dev/null || true - fi - } - trap openclaw_cleanup EXIT - - # 1) Trainer in the background; its log carries the gateway address. - "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" >"$TRAIN_LOG" 2>&1 & - TRAINER_PID=$! - - GW=$(wait_gateway "$TRAIN_LOG" "$TRAINER_PID" trainer) || exit 1 - echo "inference gateway: $GW" - - # 2) start_session (admin) -> per-session key. 429 ("no capacity") is - # retried: workers only register once the trainer is fully up. - SESS_KEY="" - for attempt in $(seq 1 24); do - BODY=$(http_post "$GW" rl/start_session "$OC_ADMIN_KEY" '{"task_id":"ci-openclaw-smoke"}' 60) - SESS_KEY=$(printf '%s' "$BODY" | json_pick '["api_key"]') - [[ -n "$SESS_KEY" ]] && break - echo "[session attempt $attempt] no session yet" - sleep 5 - done - [[ -n "$SESS_KEY" ]] || { echo "start_session never returned a key"; tail -n 40 "$TRAIN_LOG"; exit 1; } - echo "session: $SESS_KEY" - - # 3) chat/completions under the session key - REAL generation (tokens + - # logprobs recorded for RL). Long client timeout: the NPU generates - # slowly (~1.5 tok/s), far past demo_lifecycle's hardcoded 30s. - CHAT_BODY='{"model":"default","messages":[{"role":"system","content":"You are a helpful math tutor. Show your work step by step."},{"role":"user","content":"Solve step by step: What is 12 * 15 + 3?"}],"temperature":0.7,"max_tokens":256}' - CHAT=$(http_post "$GW" chat/completions "$SESS_KEY" "$CHAT_BODY" 600) - [[ -n "$CHAT" ]] || { echo "chat/completions failed"; tail -n 40 "$TRAIN_LOG"; exit 1; } - echo "chat completion: $(printf '%s' "$CHAT" | cut -c1-200)" - - # 4) set_reward (session key), then refresh: start_session with the key - # ends the session and exports the trajectory -> the trainer's - # _OnlineAgent unblocks and trains its single step. - REWARD=$(http_post "$GW" rl/set_reward "$SESS_KEY" '{"reward":1.0}' 60) - [[ -n "$REWARD" ]] || { echo "set_reward failed"; tail -n 40 "$TRAIN_LOG"; exit 1; } - echo "reward applied: $REWARD" - - REFRESHED="" - for attempt in $(seq 1 12); do - BODY=$(http_post "$GW" rl/start_session "$OC_ADMIN_KEY" \ - "{\"task_id\":\"ci-openclaw-smoke\",\"api_key\":\"$SESS_KEY\"}" 130) - NEW_KEY=$(printf '%s' "$BODY" | json_pick '["api_key"]') - [[ -n "$NEW_KEY" ]] && { REFRESHED=1; echo "session refreshed (key: $NEW_KEY)"; break; } - echo "[refresh attempt $attempt] pipeline still cycling" - sleep 10 - done - [[ -n "$REFRESHED" ]] || { echo "refresh never succeeded; trainer may be stuck"; tail -n 40 "$TRAIN_LOG"; exit 1; } - - # 5) Trainer trains its single step and exits (bounded). - wait_bounded "$TRAINER_PID" 1200 "$TRAIN_LOG" - if (( WAIT_TIMED_OUT )); then - run_rc=1 - echo "trainer wait timed out; tail of $TRAIN_LOG:" - tail -n 120 "$TRAIN_LOG" - elif wait "$TRAINER_PID"; then - echo "openclaw online-rl loop completed" - else - run_rc=$? - echo "trainer exited rc=$run_rc; tail of $TRAIN_LOG:" - tail -n 120 "$TRAIN_LOG" - fi - ;; - examples/camel/train.py) - # train.py defines its config dataclass (AgentRLConfig) at module scope. - # Whether run as a plain script or via `python -m`, the class's __module__ - # is "__main__" (verified), so the engine worker cannot import it and - # deserialize_value degrades the config to a plain dict -> config.seed - # AttributeError. Importing the file as a module instead gives the class - # the path "examples.camel.train.AgentRLConfig" (importable in the worker, - # repo root is on PYTHONPATH) and we call main() explicitly. - "$PYTHON" -c "import sys; from examples.camel.train import main; main(sys.argv[1:])" "${EXTRA_ARGS[@]}" || run_rc=$? - ;; examples/experimental/inference_service/online_rollout.py) # Rollout-only online example: online_rollout.py starts the gateway then # BLOCKS in rollout_batch(data=None) waiting for EXTERNAL traffic. We @@ -548,195 +263,6 @@ PY tail -n 120 "$ROLLOUT_LOG" fi ;; - examples/experimental/inference_service/human_in_the_loop_demo.py) - # The demo brings its own driver (launches online_rollout.py itself, - # temporarily points ~/.zeroclaw/config.toml at the local gateway, runs - # 4 HITL rounds via the zeroclaw CLI, then checks the rollout log). Its - # exit code is LOOSE though: zeroclaw failures are caught (empty answer, - # reward 0) and a missing "Rollout complete" only prints a warning - # before "Demo finished" + exit 0. Make the pass criterion strict: - # exit 0 AND "Rollout complete" in the demo output (on success it echoes - # the rollout log tail incl. "Databatch detected"). - HITL_LOG="$PWD/hitl_demo.log" - # Diagnose the zeroclaw build BEFORE the demo swallows its stderr - # (check_output discards the captured output on failure): run the demo's - # exact invocation shape against the placeholder config - # (custom:http://localhost:1). rc=2 + usage text = the installed CLI - # lacks the demo's interface - zeroclaw.com's installer ships the - # OFFICIAL build, while the demo targets the dhh1995 fork. A connection - # error instead means the syntax is fine. - # Dump the official CLI surface first: version (for later pinning) and - # whether `agent -m` / --session-state-file exist at all. - { zeroclaw --version; echo "--------"; zeroclaw --help; echo "--------"; zeroclaw agent --help; } \ - >"$PWD/zc_help.log" 2>&1 || true - echo "--- zeroclaw version / --help / agent --help (head -n 60) ---" - head -n 60 "$PWD/zc_help.log" 2>/dev/null || true - timeout 30 zeroclaw agent -m "probe" \ - --session-state-file "$PWD/zc_probe.json" >"$PWD/zc_probe.log" 2>&1 \ - || echo "zeroclaw probe rc=$? (usage error = wrong build; connection error = syntax OK)" - cat "$PWD/zc_probe.log" 2>/dev/null || true - demo_rc=0 - "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" >"$HITL_LOG" 2>&1 || demo_rc=$? - if (( demo_rc == 0 )) && grep -q 'Rollout complete' "$HITL_LOG"; then - echo "hitl demo completed" - grep 'Rollout complete' "$HITL_LOG" | tail -1 - else - run_rc=1 - echo "hitl demo failed (rc=$demo_rc, or no 'Rollout complete' - hollow run?); tail of $HITL_LOG:" - tail -n 80 "$HITL_LOG" - fi - ;; - examples/tau2/train.py) - # Tau2 keeps its REAL user simulator (solo_mode=false): host the user - # LLM locally on the runner's 2nd card with the SAME weights (zero extra - # download) and point econfig.user_llm_base_url at it (the overlay sets - # the URL). The trainer is pinned to card 0 via npu_devices, so the two - # never collide. Multiple --served-model-name aliases absorb whatever - # model string the tau2 client sends (econfig.user_llm uses litellm - # "openai/..." naming) - vLLM matches the model field strictly. - tau2_cleanup() { - if [[ -n "${USER_SIM_PID:-}" ]] && kill -0 "$USER_SIM_PID" 2>/dev/null; then - kill "$USER_SIM_PID" 2>/dev/null || true - fi - } - trap tau2_cleanup EXIT - - # 1) User-simulator vLLM on physical device 1 via the shared helper - # (sets USER_SIM_PID/USER_SIM_LOG; the EXIT trap above kills it). - start_user_sim "$PWD/tau2_user_sim.log" || exit 1 - - # 2) Trainer in the foreground; its exit code is authoritative (the - # user-sim server is torn down by the EXIT trap). - tau2_rc=0 - "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" || tau2_rc=$? - if (( tau2_rc == 0 )); then - echo "tau2 training completed" - else - run_rc=$tau2_rc - echo "tau2 trainer exited rc=$tau2_rc; user-sim tail:" - tail -n 40 "$USER_SIM_LOG" - fi - ;; - examples/agent_service/run_agent_service.py) - # The Agent Service hosts NO model itself: ClaudeAgent shells out via - # claude-agent-sdk, whose ClaudeSDKClient spawns the Claude Code CLI - - # the API traffic goes wherever ANTHROPIC_BASE_URL points. Point it at a - # SIDECAR online_rollout stack: its v2 gateway serves the Anthropic - # /v1/messages route and the data proxy translates Anthropic->OpenAI onto - # the local vLLM, so no real Anthropic key is needed. The service's own - # stdin loop is interactive: hold it open with a FIFO writer, drive ONE - # /v1/responses request (the shape documented in the example README), - # then close the FIFO -> EOF -> clean exit 0 = pass. The sidecar's - # rollout_batch never completes (no RL sessions are driven); it is just - # the LLM backend and is killed at the end. - AS_ADMIN_KEY="areal-agent-admin" - AS_SIDECAR_KEY="sk-test123456" - AS_SVC_LOG="$PWD/agent_service.log" - AS_ROLLOUT_LOG="$PWD/agent_service_rollout.log" - AS_FIFO="$PWD/agent_service_stdin.fifo" - - agent_service_cleanup() { - exec 3>&- 2>/dev/null || true - if [[ -n "${AS_SVC_PID:-}" ]] && kill -0 "$AS_SVC_PID" 2>/dev/null; then - kill "$AS_SVC_PID" 2>/dev/null || true - fi - if [[ -n "${AS_ROLLOUT_PID:-}" ]] && kill -0 "$AS_ROLLOUT_PID" 2>/dev/null; then - kill "$AS_ROLLOUT_PID" 2>/dev/null || true - fi - } - trap agent_service_cleanup EXIT - - # 1) Sidecar: online_rollout.py (vLLM + gateway serving /v1/messages, - # registered model name "default"). - "$PYTHON" examples/experimental/inference_service/online_rollout.py \ - --config examples/experimental/inference_service/online_rollout.yaml \ - rollout.backend=vllm:d1p1t1 \ - actor.backend=fsdp:d1p1t1 \ - actor.path="$AREAL_MODEL_PATH" \ - train_dataset.batch_size=1 \ - vllm.gpu_memory_utilization=0.25 \ - vllm.max_model_len=16384 \ - gconfig.max_new_tokens=256 \ - rollout.setup_timeout=1800 \ - >"$AS_ROLLOUT_LOG" 2>&1 & - AS_ROLLOUT_PID=$! - - AS_INF_GW=$(wait_gateway "$AS_ROLLOUT_LOG" "$AS_ROLLOUT_PID" sidecar) || exit 1 - echo "sidecar inference gateway: $AS_INF_GW" - - # 2) The Agent Service, stdin held open by a FIFO writer on fd 3 (a plain - # background run would hit EOF immediately and exit). Claude Code CLI - # env: traffic -> the sidecar gateway; nonessential phones-home - # disabled (those hosts are unreachable from the CN runners and would - # stall the CLI); generous API timeout for the slow NPU generation. - mkfifo "$AS_FIFO" - ANTHROPIC_BASE_URL="$AS_INF_GW" \ - ANTHROPIC_API_KEY="$AS_SIDECAR_KEY" \ - CLAUDE_MODEL=default \ - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 \ - DISABLE_TELEMETRY=1 \ - API_TIMEOUT_MS=600000 \ - "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" <"$AS_FIFO" >"$AS_SVC_LOG" 2>&1 & - AS_SVC_PID=$! - exec 3>"$AS_FIFO" - - # 3) Wait for the service gateway ("Gateway: http://..."), then /health. - AS_AGENT_GW="" - for i in $(seq 1 60); do - AS_AGENT_GW=$(grep 'Gateway:' "$AS_SVC_LOG" 2>/dev/null | tail -1 \ - | grep -oE 'https?://[A-Za-z0-9._:-]+' | tail -1 || true) - [[ -n "$AS_AGENT_GW" ]] && break - kill -0 "$AS_SVC_PID" 2>/dev/null || break - sleep 5 - done - if [[ -n "$AS_AGENT_GW" ]]; then - for i in $(seq 1 24); do - http_healthy "${AS_AGENT_GW}/health" && break - kill -0 "$AS_SVC_PID" 2>/dev/null || break - sleep 5 - done - fi - if [[ -z "$AS_AGENT_GW" ]] || ! http_healthy "${AS_AGENT_GW}/health"; then - echo "agent service gateway not healthy; tail of $AS_SVC_LOG:" - tail -n 120 "$AS_SVC_LOG" - exit 1 - fi - echo "agent service gateway: $AS_AGENT_GW" - - # 4) ONE conversation through the service (the README's documented - # request shape). The worker spawns the Claude Code CLI, whose API - # calls land on the sidecar's /v1/messages -> local vLLM. - AS_RESP=$(http_post "$AS_AGENT_GW" v1/responses "$AS_ADMIN_KEY" \ - '{"input":[{"type":"message","content":"Say hello in one sentence."}],"model":"claude-agent","user":"ci-agent-service"}' 600) - if [[ -z "$AS_RESP" ]] || ! printf '%s' "$AS_RESP" | grep -q 'completed'; then - run_rc=1 - echo "agent turn failed; response head + service tail:" - printf '%s\n' "$AS_RESP" | cut -c1-400 - tail -n 80 "$AS_SVC_LOG" - else - echo "agent reply: $(printf '%s' "$AS_RESP" | grep -oE '"text": *"[^"]{0,120}' | head -1)" - fi - - # 5) Close the FIFO writer -> the interactive loop sees EOF -> cleanup + - # exit 0. The service's exit code is the pass criterion. - exec 3>&- - wait_bounded "$AS_SVC_PID" 120 "$AS_SVC_LOG" - if (( WAIT_TIMED_OUT == 0 )); then - as_rc=0 - wait "$AS_SVC_PID" || as_rc=$? - if (( as_rc == 0 )) && (( run_rc == 0 )); then - echo "agent_service completed" - else - run_rc=${as_rc:-1} - echo "agent service exited rc=$as_rc; tail of $AS_SVC_LOG:" - tail -n 120 "$AS_SVC_LOG" - fi - else - run_rc=1 - echo "agent service did not exit after EOF; tail of $AS_SVC_LOG:" - tail -n 120 "$AS_SVC_LOG" - fi - ;; examples/experimental/inference_service/tau2_rollout.py) # Rollout-only tau2: the SAME real agent + user simulator as the # training entry (Tau2AgentWorkflow, airline domain), but driven through diff --git a/examples-guards/areal/scripts/setup_example.sh b/examples-guards/areal/scripts/setup_example.sh index d380a9a89..273df94b7 100644 --- a/examples-guards/areal/scripts/setup_example.sh +++ b/examples-guards/areal/scripts/setup_example.sh @@ -16,67 +16,20 @@ fi PROFILE="$1" -SUPPORTED_PROFILES="areal-vlm-grpo areal-vlm-mt-grpo areal-vlm-sft areal-clevr-sft areal-tir-grpo areal-scaffold-grpo areal-agents-grpo areal-math-grpo areal-math-sft areal-math-aime areal-math-boba areal-countdown-grpo areal-hermes-grpo areal-openclaw-grpo areal-camel-grpo areal-online-rollout areal-hitl-demo areal-tau2-grpo areal-agent-service areal-align" -AIME_PREP=0 -BOBA_PREP=0 -COUNTDOWN_PREP=0 -HHRLHF_PREP=0 -TORL_PREP=0 -SCAFFOLD_PREP=0 -CLEVR_SUBSET_PREP=0 -HERMES_PREP=0 -OPENCLAW_PREP=0 -CAMEL_PREP=0 -HITL_PREP=0 +SUPPORTED_PROFILES="areal-math-sft areal-swe-sft areal-online-rollout areal-tau2-grpo" TAU2_PREP=0 -AGENT_SERVICE_PREP=0 case "$PROFILE" in - areal-vlm-grpo) MODEL_ID="Qwen/Qwen2.5-VL-3B-Instruct" ;; - areal-vlm-mt-grpo) MODEL_ID="Qwen/Qwen3-VL-2B-Instruct" ;; - areal-vlm-sft) MODEL_ID="Qwen/Qwen3-VL-2B-Instruct" ;; - # clevr_count_70k_sft.py: see section 10 (local subset; the full 70k train - # split blows the data worker when the SFT loader materializes pixel_values). - areal-clevr-sft) MODEL_ID="Qwen/Qwen2.5-VL-3B-Instruct"; CLEVR_SUBSET_PREP=1 ;; - # tir/train_tir.py: see section 8 (fixture staging; the loader's own - # GitHub download is unreachable from the CI job containers). - areal-tir-grpo) MODEL_ID="Qwen/Qwen2.5-Math-1.5B"; TORL_PREP=1 ;; - # gsm8k_rlvr_scaffolding.py: 1.5B model + section 9 (model alias symlink). - areal-scaffold-grpo) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; SCAFFOLD_PREP=1 ;; - areal-agents-grpo) MODEL_ID="Qwen/Qwen2-1.5B-Instruct" ;; - # gsm8k_rl.py and gsm8k_eval.py share the same model. - areal-math-grpo) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct" ;; areal-math-sft) MODEL_ID="Qwen/Qwen3-1.7B" ;; - areal-math-aime) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; AIME_PREP=1 ;; - areal-math-boba) MODEL_ID="deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"; BOBA_PREP=1 ;; - areal-countdown-grpo) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; COUNTDOWN_PREP=1 ;; - # hermes/train.py: online RL; agent runtime package installed in - # section 1c, orchestration in run_example.sh's hermes branch. - areal-hermes-grpo) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; HERMES_PREP=1 ;; - # openclaw/train.py: online RL through the OpenAI-compatible proxy gateway; - # no agent runtime package to install (demo_lifecycle.py is pure HTTP). - # No source patch (no-patch policy) - demo_lifecycle.py runs as shipped. - # Reuses hermes' model. - areal-openclaw-grpo) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; OPENCLAW_PREP=1 ;; - # camel/train.py: camel-ai ChatAgent GRPO. Needs the undeclared camel-ai - # package (section 1d) and a `python -m` launch (run_example.sh camel branch). - areal-camel-grpo) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; CAMEL_PREP=1 ;; + # swe/train_sft.py: dataset path must contain an "swe" token to route + # into the SWE pipeline (fixture swe_sft_pairs.jsonl via FIXTURE_DIR). + areal-swe-sft) MODEL_ID="Qwen/Qwen3-1.7B" ;; # online_rollout.py: rollout-only online demo; no extra runtime to install # (drive is run_example.sh's online_rollout branch). areal-online-rollout) MODEL_ID="Qwen/Qwen3-1.7B" ;; - # human_in_the_loop_demo.py: zeroclaw-driven HITL demo; section 1e installs - # the CLI and plants its home config skeleton. - areal-hitl-demo) MODEL_ID="Qwen/Qwen3-0.6B"; HITL_PREP=1 ;; - # tau2/train.py: multi-turn customer-service RL (real user simulator; the - # user LLM is hosted locally by run_example.sh's tau2 branch). Section 1f - # installs the tau2-bench fork + data. + # tau2_rollout.py: rollout-only tau2 (agent + user simulator); section 1f + # installs the tau2-bench fork + data, run_example.sh's tau2_rollout branch + # hosts the user-sim vLLM on card 1. areal-tau2-grpo) MODEL_ID="Qwen/Qwen3-1.7B"; TAU2_PREP=1 ;; - # agent_service/run_agent_service.py: Claude Agent SDK micro-service; the - # Claude Code CLI it spawns is installed in section 1g; run_example.sh's - # agent-service branch points its API traffic at a local sidecar gateway. - areal-agent-service) MODEL_ID="Qwen/Qwen3-0.6B"; AGENT_SERVICE_PREP=1 ;; - # hhrlhf_dpo.py: DPO pipeline is model-agnostic; upstream yaml's 7B swapped - # for 1.5B (7B fp32 init OOMs the 32GB cards in the pool, 15GB per run). - areal-align) MODEL_ID="Qwen/Qwen2.5-1.5B-Instruct"; HHRLHF_PREP=1 ;; *) echo "unknown profile: ${PROFILE} (supported: ${SUPPORTED_PROFILES})" >&2 exit 1 @@ -89,8 +42,8 @@ if [[ -z "${TARGET_ROOT:-}" || -z "${GITHUB_WORKSPACE:-}" || -z "${GITHUB_ENV:-} fi # ------------------------------------------------------- -# Shared helpers: mirror pip installs, shallow clones, tarball -# fetches - one place for the mirror URLs and CI timeout budget. +# Shared helpers: mirror pip installs and shallow clones - one place +# for the mirror URLs. # ------------------------------------------------------- pip_tuna() { uv pip install --system --index-url https://pypi.tuna.tsinghua.edu.cn/simple "$@" @@ -101,17 +54,18 @@ pip_aliyun() { } shallow_clone() { - git clone --depth 1 --branch "$2" "$1" "$3" -} - -# Tarball fetch with a long budget and one resume retry: the zeroclaw -# release comes from github.com, whose throughput from the CN runners is -# marginal - 15.5MB once needed >600s and got cut at 91%. -C - resumes -# the partial file instead of restarting from byte 0. -fetch_tarball() { - curl -fsSL --connect-timeout 30 --max-time 1800 -C - "$1" -o "$2" \ - || curl -fsSL --connect-timeout 30 --max-time 1800 -C - "$1" -o "$2" \ - || { echo "failed to download $1" >&2; exit 2; } + local url=$1 branch=$2 dest=$3 attempt + # CI 出口经 gh-proxy/squid,长传输偶发 curl 18/early EOF 截断; + # 失败时目录可能残留,重试前清理,最多 3 次。 + for attempt in 1 2 3; do + rm -rf "$dest" + if git clone --depth 1 --branch "$branch" "$url" "$dest"; then + return 0 + fi + echo "shallow_clone failed (attempt ${attempt}/3): ${url}@${branch}" >&2 + sleep 15 + done + return 1 } # ------------------------------------------------------- @@ -161,83 +115,6 @@ VLLM_TARGET_DEVICE=empty uv pip install --system --no-deps --no-build-isolation -e "$VLLM_SRC" python -c "import torch, torch_npu, vllm, vllm_ascend; from vllm.entrypoints.openai.utils import validate_json_request; print(f'vllm {vllm.__version__} + vllm-ascend 0.22.1rc1 layout OK; torch {torch.__version__}, torch_npu {torch_npu.__version__}')" -# ------------------------------------------------------- -# 1c. hermes profile: the agent runtime package. The model is the -# standard Qwen2.5-1.5B-Instruct (downloaded by section 3); hermes-agent -# itself is model-agnostic - it drives whatever OpenAI-compatible endpoint -# the rollout engine serves. The Agent Service imports -# examples.hermes.hermes.HermesAgent from the target checkout. -# Index: tsinghua (not aliyun) - aliyun's PEP 658 .metadata sidecar for -# hermes-agent's python-dotenv dep 404s intermittently; tuna mirrors the -# metadata reliably (same mirror as transformers-quick-start.yml). -# ------------------------------------------------------- -if [[ "$HERMES_PREP" == 1 ]]; then - pip_tuna hermes-agent==0.19.0 -fi - -# ------------------------------------------------------- -# 1d. camel profile: the camel-ai agent runtime. -# examples/camel/train.py imports camel.agents.ChatAgent, and -# areal.experimental.camel.openai_model imports camel.logger / camel.messages / -# camel.models.base_model / camel.types / camel.utils; camel-ai is NOT an -# AReaL-declared dependency, so it is installed here. camel-ai goes in with -# --no-deps so the job's NPU torch/vllm stack is never touched (camel is -# API-based); its lighter runtime deps come in separately (none pull torch). -# ------------------------------------------------------- -if [[ "$CAMEL_PREP" == 1 ]]; then - pip_tuna --no-deps camel-ai - pip_tuna tiktoken docstring-parser colorama python-dotenv jsonschema -fi - -# ------------------------------------------------------- -# 1e. hitl-demo profile: the zeroclaw chat client + its home -# config skeleton. human_in_the_loop_demo.py drives its own traffic through -# the zeroclaw CLI (`zeroclaw agent -m ... --session-state-file ...`), so the -# binary must be on PATH. The demo's FIRST action on the config is -# shutil.copy2(~/.zeroclaw/config.toml, .demo_bak) -> FileNotFoundError if the -# file is missing, and its regex patch silently skips a missing -# default_provider line - so plant a minimal skeleton in the runner home dir -# (the demo backs it up and restores it on exit; AReaL source is untouched). -# If zeroclaw.com turns out unreachable from the CN runners, switch to the -# fork's GitHub Releases aarch64-linux tarball. -# ------------------------------------------------------- -if [[ "$HITL_PREP" == 1 ]]; then - # PINNED to v0.6.0: the last release line whose CLI matches the demo's - # hardcoded invocation `zeroclaw agent -m ... --session-state-file ...`. - # v0.6.0's agent subcommand has NO --agent flag and its Providers list - # documents `custom:` - the inline OpenAI-compatible provider the - # demo's config patcher writes. v0.7+ introduced the agent-alias system - # and made --agent REQUIRED ("Required - there is no default agent"), - # so the current official build (zeroclaw.com / releases/latest, v0.8.x) - # rejects the demo's invocation with a usage error (observed rc=2). - # zeroclaw.com's installer has no version flag, so fetch the pinned - # release tarball directly - github release downloads are reachable from - # the CN runners (zeroclaw.com's own installer fetches from there). - # /usr/local/bin is on the default PATH, so no PATH plumbing is needed. - ZC_VERSION=v0.6.0 - ZC_ASSET="zeroclaw-aarch64-unknown-linux-gnu.tar.gz" - ZC_URL="https://github.com/zeroclaw-labs/zeroclaw/releases/download/${ZC_VERSION}/${ZC_ASSET}" - ZC_TMP="$(mktemp -d)" - # a 404 here usually means the aarch64 asset is missing in $ZC_VERSION - fetch_tarball "$ZC_URL" "$ZC_TMP/$ZC_ASSET" - tar -xzf "$ZC_TMP/$ZC_ASSET" -C "$ZC_TMP" - [[ -x "$ZC_TMP/zeroclaw" ]] || { echo "$ZC_ASSET did not contain a zeroclaw binary" >&2; exit 2; } - install -m 0755 "$ZC_TMP/zeroclaw" /usr/local/bin/zeroclaw - rm -rf "$ZC_TMP" - zeroclaw --version || { echo "zeroclaw failed to run" >&2; exit 2; } - # Same flat keys the demo's patcher regexes target (plus workspace_dir, - # mirroring what v0.6.0's own installer writes); the demo rewrites - # default_provider/api_key to the live gateway on start. - mkdir -p "$HOME/.zeroclaw" - cat > "$HOME/.zeroclaw/config.toml" <<'TOML' -default_provider = "custom:http://localhost:1" -default_model = "default" -api_key = "placeholder" -workspace_dir = "/tmp/zeroclaw-workspace" -TOML - echo "planted minimal ~/.zeroclaw/config.toml skeleton" -fi - # ------------------------------------------------------- # 1f. tau2 profile: the tau2-bench fork (async orchestrator # + user simulator) and its task data. TAU2_DATA_DIR must reach the trainer @@ -254,42 +131,6 @@ if [[ "$TAU2_PREP" == 1 ]]; then echo "tau2-bench fork installed; TAU2_DATA_DIR=$TAU2_DIR/data" fi -# ------------------------------------------------------- -# 1g. agent-service profile: the Claude Agent SDK stack. -# examples/agent_service's ClaudeAgent shells out through claude-agent-sdk, -# whose ClaudeSDKClient spawns the Claude Code CLI (an npm package) - so -# besides the pip package we need Node >= 18 and the CLI itself. The NPU -# image ships no node, so install a pinned Node from the npmmirror CDN -# (CN-reachable, unlike nodejs.org) and the CLI from the npmmirror -# registry. The API traffic is pointed at the LOCAL gateway by -# run_example.sh's agent-service branch (ANTHROPIC_BASE_URL) - no real -# Anthropic key is ever used. -# ------------------------------------------------------- -if [[ "$AGENT_SERVICE_PREP" == 1 ]]; then - pip_tuna claude-agent-sdk - AS_NEED_NODE=1 - if command -v node >/dev/null 2>&1; then - AS_NODE_MAJOR="$(node --version | sed 's/^v\([0-9]*\).*/\1/')" - if (( AS_NODE_MAJOR >= 18 )); then AS_NEED_NODE=0; fi - fi - if [[ "$AS_NEED_NODE" == 1 ]]; then - AS_NODE_VERSION=v22.14.0 - AS_NODE_DIR="/usr/local/node-${AS_NODE_VERSION}" - fetch_tarball \ - "https://registry.npmmirror.com/-/binary/node/${AS_NODE_VERSION}/node-${AS_NODE_VERSION}-linux-arm64.tar.gz" \ - /tmp/as-node.tar.gz - mkdir -p "$AS_NODE_DIR" - tar -xzf /tmp/as-node.tar.gz -C "$AS_NODE_DIR" --strip-components=1 - export PATH="$AS_NODE_DIR/bin:$PATH" - echo "PATH=$PATH" >> "$GITHUB_ENV" - fi - node --version || { echo "node unavailable for the claude CLI" >&2; exit 2; } - npm install -g @anthropic-ai/claude-code \ - --registry=https://registry.npmmirror.com \ - || { echo "claude code CLI install failed" >&2; exit 2; } - claude --version || { echo "claude CLI failed to run" >&2; exit 2; } -fi - # ------------------------------------------------------- # 2. Runtime Environment setup. # ------------------------------------------------------- @@ -337,9 +178,8 @@ export AREAL_MODEL_ID="$MODEL_ID" # The resolved path is captured into the AREAL_MODEL_PATH shell variable (tee # keeps the diagnostics streaming into the job log) and written to GITHUB_ENV -# for the run step. Later sections of this script - countdown's tokenizer -# (section 6) and the scaffolding model alias (section 9) - consume it, so it -# must be the real on-disk download dir. +# for the run step (overlay args reference AREAL_MODEL_PATH), so it must be +# the real on-disk download dir. AREAL_MODEL_PATH=$(python3 <<'PY' | tee /dev/stderr | sed -n 's/^AREAL_MODEL_PATH=//p' | tail -n 1 import os import subprocess @@ -391,215 +231,3 @@ PY echo "model resolution printed no AREAL_MODEL_PATH line" >&2 exit 2 } - -# ------------------------------------------------------- -# 4. AIME dataset + derived config (areal-math-aime). -# ------------------------------------------------------- -# areal/dataset/aime.py loads /aime_train.parquet and -# /aime_test.parquet from a LOCAL directory (upstream ships no aime -# config; the loader takes no HF repo id). Convert the public MathArena -# parquets (same flow as examples/distillation/mopd/README.md), then derive -# the GRPO config from gsm8k_grpo_npu.yaml: point the datasets at the local -# dir and set rollout.agent to null (aime_rl.py runs the standard -# RLVRWorkflow, not an agent workflow). The generated config is exposed to -# overlay args via AREAL_AIME_CONFIG. -if [[ "$AIME_PREP" == 1 ]]; then -python3 <<'PY' -import os - -import pandas as pd -import yaml -from huggingface_hub import hf_hub_download - -workspace = os.environ["GITHUB_WORKSPACE"] -target_root = os.environ["TARGET_ROOT"] -aime_dir = os.path.join(workspace, "areal_data", "aime") -os.makedirs(aime_dir, exist_ok=True) - -# hf_hub_download honors HF_ENDPOINT set in section 3. -for repo, dst in [ - ("MathArena/aime_2025", "aime_train.parquet"), - ("MathArena/aime_2026", "aime_test.parquet"), -]: - src = hf_hub_download( - repo_id=repo, repo_type="dataset", - filename="data/train-00000-of-00001.parquet", - ) - df = pd.read_parquet(src).rename(columns={"problem": "question"}) - df = df[["question", "answer"]] - df.to_parquet(os.path.join(aime_dir, dst)) - print(f"prepared {dst}: {len(df)} rows", flush=True) - -base = os.path.join(target_root, "examples", "math", "gsm8k_grpo_npu.yaml") -with open(base, encoding="utf-8") as fh: - cfg = yaml.safe_load(fh) -cfg["experiment_name"] = "aime-grpo" -cfg["train_dataset"]["path"] = aime_dir -cfg["valid_dataset"]["path"] = aime_dir -config_path = os.path.join(aime_dir, "aime_grpo_npu.yaml") -with open(config_path, "w", encoding="utf-8") as fh: - yaml.safe_dump(cfg, fh, sort_keys=False, allow_unicode=True) - -with open(os.environ["GITHUB_ENV"], "a") as fh: - fh.write(f"AREAL_AIME_CONFIG={config_path}\n") -print(f"generated {config_path}", flush=True) -PY -fi - -# ------------------------------------------------------- -# 5. BoBa dataset (areal-math-boba). -# ------------------------------------------------------- -# boba_grpo.py loads train_dataset.path as a LOCAL jsonl file -# (load_dataset("json", data_files=path)); the upstream yaml ships the HF -# repo id, which would fail with FileNotFoundError. Pre-download the public -# AReaL-boba-106k.jsonl and expose its local path via AREAL_BOBA_DATA. -if [[ "$BOBA_PREP" == 1 ]]; then -python3 <<'PY' -import os -import shutil - -from huggingface_hub import hf_hub_download - -src = hf_hub_download( - repo_id="inclusionAI/AReaL-boba-Data", repo_type="dataset", - filename="AReaL-boba-106k.jsonl", -) -dst = os.path.join( - os.environ["GITHUB_WORKSPACE"], "areal_data", "boba", "AReaL-boba-106k.jsonl" -) -os.makedirs(os.path.dirname(dst), exist_ok=True) -shutil.copyfile(src, dst) -with open(os.environ["GITHUB_ENV"], "a") as fh: - fh.write(f"AREAL_BOBA_DATA={dst}\n") -print(f"downloaded boba dataset to {dst}", flush=True) -PY -fi - -# ------------------------------------------------------- -# 6. Countdown dataset (areal-countdown-grpo). -# ------------------------------------------------------- -# countdown.py writes ./data/countdown/qwen/*.jsonl relative to CWD; run it -# from TARGET_ROOT so the generated files land where train_config.yaml's -# default train_dataset.path=data/countdown/qwen/train_e.jsonl expects them -# (run_example.sh also cds to TARGET_ROOT). The default generation size is -# 500k samples; 32/8 is plenty for the 1-step smoke. The tokenizer comes -# from the model predownloaded in section 3, so generation is offline. -if [[ "$COUNTDOWN_PREP" == 1 ]]; then - MODEL_DIR="$AREAL_MODEL_PATH" - mkdir -p "$TARGET_ROOT/data/countdown/qwen" - (cd "$TARGET_ROOT" && python3 examples/countdown/countdown.py \ - --num_samples 32 --eval_size 8 --tokenizer_path "$MODEL_DIR") -fi - -# ------------------------------------------------------- -# 7. HH-RLHF dataset (areal-align: hhrlhf_dpo.py / hhrlhf_rw.py). -# ------------------------------------------------------- -# Anthropic/hh-rlhf is public (NOT gated), but its jsonl.gz files live in -# per-subset subdirectories (harmless-base/, helpful-base/, ...), so the -# upstream loader's load_dataset(path, split=...) without a config name -# fails with a multi-config error. Work around it boba-style: download the -# harmless-base subset online (via HF_ENDPOINT) into a flat local -# dir; load_dataset(, split="train"/"test") then infers splits from -# the file names. Exposed to overlay args via AREAL_HHRLHF_DATA. -# NOTE: the dir is deliberately named "hh-rlhf" 鈥?areal's dataset dispatch -# matches the literal substring "hh-rlhf" in the path; a dir named -# "hhrlhf" falls through to the load_from_disk fallback and errors out. -if [[ "$HHRLHF_PREP" == 1 ]]; then -python3 <<'PY' -import os -import shutil - -from huggingface_hub import hf_hub_download - -dst_dir = os.path.join(os.environ["GITHUB_WORKSPACE"], "areal_data", "hh-rlhf") -os.makedirs(dst_dir, exist_ok=True) -for filename in ("train.jsonl.gz", "test.jsonl.gz"): - src = hf_hub_download( - repo_id="Anthropic/hh-rlhf", repo_type="dataset", - filename=f"harmless-base/{filename}", - ) - shutil.copyfile(src, os.path.join(dst_dir, filename)) -with open(os.environ["GITHUB_ENV"], "a") as fh: - fh.write(f"AREAL_HHRLHF_DATA={dst_dir}\n") -print(f"prepared hh-rlhf harmless-base at {dst_dir}", flush=True) -PY -fi - -# ------------------------------------------------------- -# 8. ToRL dataset (areal-tir-grpo): fixture staging. -# ------------------------------------------------------- -# The torl_data loader downloads its parquets from github.com at load time -# (areal/dataset/torl_data.py), but the CI job containers cannot reach -# github.com (mainland network, no proxy; the runner agent pulls code on the -# host side). No HF/ModelScope mirror of ToRL exists (ModelScope 404; -# upstream ships GitHub only), so the two tiny parquets are vendored as -# fixtures - the documented last resort. prepare_torl_data() skips its -# download once /tmp/areal/torl_data/_SUCCESS exists, so staging the files -# plus the flag here makes the loader fully offline. The data service worker -# runs in the same container, so /tmp is shared with the run step. -# Fixture pin (GAIR-NLP/ToRL @ main): train.parquet blob 57bd7d5d (6035244 -# bytes), test.parquet blob e8ca20fd (51798 bytes). -if [[ "$TORL_PREP" == 1 ]]; then - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - TORL_FIXTURE="$SCRIPT_DIR/../fixtures" - for f in train.parquet test.parquet; do - [[ -s "$TORL_FIXTURE/$f" ]] || { - echo "missing fixture: $TORL_FIXTURE/$f (see section 8 comment for source)" >&2 - exit 2 - } - done - mkdir -p /tmp/areal/torl_data - cp "$TORL_FIXTURE/train.parquet" "$TORL_FIXTURE/test.parquet" /tmp/areal/torl_data/ - : > /tmp/areal/torl_data/_SUCCESS - echo "staged torl fixtures at /tmp/areal/torl_data (loader download skipped)" -fi - -# ------------------------------------------------------- -# 9. Scaffolding model alias (areal-scaffold-grpo). -# ------------------------------------------------------- -# examples/scaffolding hardcodes model="default" in its OpenAI-compatible -# worker (workflow.py:102, a TRT-LLM convention), while AReaL's vLLM server -# serves under the model path - requests 404 and every rollout comes back -# empty, which then trips the "total loss weight must be positive" assert. -# vLLMConfig has no served_model_name passthrough (strict dataclass, unknown -# keys are rejected at the structured merge), so serve the model under the -# literal name "default" instead: vLLM uses the raw --model string as the -# served model name, so a relative "default" path resolving to the model dir -# makes the name match. The server subprocess inherits the run CWD -# (TARGET_ROOT), hence the symlink location. -if [[ "$SCAFFOLD_PREP" == 1 ]]; then - MODEL_DIR="$AREAL_MODEL_PATH" - [[ -d "$MODEL_DIR" ]] || { echo "model dir missing: $MODEL_DIR" >&2; exit 2; } - ln -sfn "$MODEL_DIR" "$TARGET_ROOT/default" - echo "linked $TARGET_ROOT/default -> $MODEL_DIR (scaffolding model alias)" -fi - -# ------------------------------------------------------- -# 10. CLEVR subset (areal-clevr-sft). -# ------------------------------------------------------- -# The clevr SFT loader materializes per-sample pixel_values across the whole -# split (clevr_count_70k.py:104-127), unlike the RL loader which only keeps -# JPEG bytes, so the 70k train split blows the data worker (host OOM / tens -# of GB of arrow cache). The example hardcodes split="train" and -# get_custom_dataset ignores train_dataset.split when the script passes a -# non-None split (dataset/__init__.py:322,328), and the loader takes no extra -# kwargs - so subset via path instead: train_dataset.path points at a dir -# whose name contains "clevr_count_70k" (dispatch substring) holding -# train.parquet (load_dataset infers the split from the file name). The -# absolute slice train[:200] downloads only the first shard (~0.5GB of the -# 10.4GB), and the parquet keeps the HF feature metadata so images stay PIL. -if [[ "$CLEVR_SUBSET_PREP" == 1 ]]; then -python3 <<'PY' -import os - -from datasets import load_dataset - -dst = os.path.join(os.environ["GITHUB_WORKSPACE"], "areal_data", "clevr_count_70k") -os.makedirs(dst, exist_ok=True) -ds = load_dataset("BUAADreamer/clevr_count_70k", split="train[:200]") -ds.to_parquet(os.path.join(dst, "train.parquet")) -with open(os.environ["GITHUB_ENV"], "a") as fh: - fh.write(f"AREAL_CLEVR_SUBSET_DIR={dst}\n") -print(f"prepared clevr subset ({len(ds)} rows) at {dst}", flush=True) -PY -fi