From 0c34dfa778dc8ead169bfe40d0f639f6002bb79e Mon Sep 17 00:00:00 2001 From: KunyangZhang <129836809+KunyangZhang@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:32:14 +0800 Subject: [PATCH] Add GSM8K examples for RLOO, REINFORCE++, GSPO and SAPO Four algorithms are implemented and listed in the supported-algorithm table but have no example: RLOO, REINFORCE++, GSPO and SAPO (their `Doc / Example` column reads `-`). This follows the "Improve Examples" item in #470. Each example mirrors examples/grpo_gsm8k/gsm8k.yaml and differs only in the experiment name and `algorithm.algorithm_type`; every other field (dataset, model, cluster, buffer, explorer, trainer) is unchanged, so the algorithm's own `default_config()` supplies the advantage/policy-loss wiring. The support tables in README.md, README_zh.md, docs/.../main.md and the dataset-perspective tables (EN and ZH) now link the new examples; the CISPO example link, which existed in the repository but was still shown as `-`, is restored as well. Verified by running the body of tests/common/config_test.py::test_all_examples_are_valid against both pre-existing and new configs (load_config + check_and_update), and by checking the resolved wiring: rloo -> advantage_fn=rloo, reinforceplusplus -> advantage_fn=reinforceplusplus/gamma=1.0, gspo -> policy_loss_fn=gspo, sapo -> policy_loss_fn=sapo (tau_pos=1.0, tau_neg=1.05). Refs #470 --- README.md | 10 +-- README_zh.md | 10 +-- docs/sphinx_doc/source/main.md | 10 +-- .../tutorial/example_dataset_perspective.md | 4 + docs/sphinx_doc/source_zh/main.md | 10 +-- .../tutorial/example_dataset_perspective.md | 4 + examples/gspo_gsm8k/README.md | 5 ++ examples/gspo_gsm8k/gsm8k.yaml | 83 +++++++++++++++++++ examples/reinforceplusplus_gsm8k/README.md | 5 ++ examples/reinforceplusplus_gsm8k/gsm8k.yaml | 83 +++++++++++++++++++ examples/rloo_gsm8k/README.md | 5 ++ examples/rloo_gsm8k/gsm8k.yaml | 83 +++++++++++++++++++ examples/sapo_gsm8k/README.md | 5 ++ examples/sapo_gsm8k/gsm8k.yaml | 83 +++++++++++++++++++ 14 files changed, 380 insertions(+), 20 deletions(-) create mode 100644 examples/gspo_gsm8k/README.md create mode 100644 examples/gspo_gsm8k/gsm8k.yaml create mode 100644 examples/reinforceplusplus_gsm8k/README.md create mode 100644 examples/reinforceplusplus_gsm8k/gsm8k.yaml create mode 100644 examples/rloo_gsm8k/README.md create mode 100644 examples/rloo_gsm8k/gsm8k.yaml create mode 100644 examples/sapo_gsm8k/README.md create mode 100644 examples/sapo_gsm8k/gsm8k.yaml diff --git a/README.md b/README.md index 23afbbe5300..6c2c96e6ea9 100644 --- a/README.md +++ b/README.md @@ -122,14 +122,14 @@ Trinity-RFT provides functionalities for users with different backgrounds and ob | DPO [[Paper](https://arxiv.org/pdf/2305.18290)] | [[HumanLike Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/dpo_humanlike)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/dpo_loss.py)] | `algorithm_type: dpo` | | CHORD 💡 [[Paper](https://arxiv.org/pdf/2508.11408)] | [[Doc](https://agentscope-ai.github.io/Trinity-RFT/en/main/tutorial/example_mix_algo.html)] [[ToolACE Example](https://github.com/agentscope-ai/Trinity-RFT/blob/main/examples/mix_chord/mix_chord_toolace.yaml)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/chord_policy_loss.py)] | `algorithm_type: mix_chord` | | REC Series 💡 [[Paper](https://arxiv.org/pdf/2509.24203)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rec_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/rec_policy_loss.py)] | `algorithm_type: rec` | -| RLOO [[Paper](https://arxiv.org/pdf/2402.14740)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | -| REINFORCE++ [[Paper](https://arxiv.org/pdf/2501.03262)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | -| GSPO [[Paper](https://arxiv.org/pdf/2507.18071)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | +| RLOO [[Paper](https://arxiv.org/pdf/2402.14740)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rloo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | +| REINFORCE++ [[Paper](https://arxiv.org/pdf/2501.03262)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/reinforceplusplus_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | +| GSPO [[Paper](https://arxiv.org/pdf/2507.18071)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/gspo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | | TOPR [[Paper](https://arxiv.org/pdf/2503.14286)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/topr_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/topr_policy_loss.py)] | `algorithm_type: topr` | | sPPO [[Paper](https://arxiv.org/pdf/2108.05828)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sppo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sppo_loss_fn.py)] | `algorithm_type: sppo` | | AsymRE [[Paper](https://arxiv.org/pdf/2506.20520)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/asymre_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/asymre_advantage.py)] | `algorithm_type: asymre` | -| CISPO [[Paper](https://arxiv.org/pdf/2506.13585)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | -| SAPO [[Paper](https://arxiv.org/pdf/2511.20347)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | +| CISPO [[Paper](https://arxiv.org/pdf/2506.13585)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/cispo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | +| SAPO [[Paper](https://arxiv.org/pdf/2511.20347)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sapo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | | On-Policy Distillation [[Blog](https://thinkingmachines.ai/blog/on-policy-distillation/)] [[Paper](https://arxiv.org/pdf/2306.13649)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/common/workflows/on_policy_distill_workflow.py)] | `algorithm_type: on_policy_distill` | | JSD (Jensen-Shannon Divergence) | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k/opd_gsm8k_jsd.yaml)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/jsd_advantage.py)] | `algorithm_type: jsd` | diff --git a/README_zh.md b/README_zh.md index f9f08a1b509..632b9c3e3da 100644 --- a/README_zh.md +++ b/README_zh.md @@ -142,14 +142,14 @@ Trinity-RFT 面向不同背景和目标的用户提供相应功能： | DPO [[论文](https://arxiv.org/pdf/2305.18290)] | [[HumanLike 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/dpo_humanlike)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/dpo_loss.py)] | `algorithm_type: dpo` | | CHORD 💡 [[论文](https://arxiv.org/pdf/2508.11408)] | [[文档](https://agentscope-ai.github.io/Trinity-RFT/zh/main/tutorial/example_mix_algo.html)] [[ToolACE 示例](https://github.com/agentscope-ai/Trinity-RFT/blob/main/examples/mix_chord/mix_chord_toolace.yaml)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/chord_policy_loss.py)] | `algorithm_type: mix_chord` | | REC Series 💡 [[论文](https://arxiv.org/pdf/2509.24203)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rec_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/rec_policy_loss.py)] | `algorithm_type: rec` | -| RLOO [[论文](https://arxiv.org/pdf/2402.14740)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | -| REINFORCE++ [[论文](https://arxiv.org/pdf/2501.03262)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | -| GSPO [[论文](https://arxiv.org/pdf/2507.18071)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | +| RLOO [[论文](https://arxiv.org/pdf/2402.14740)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rloo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | +| REINFORCE++ [[论文](https://arxiv.org/pdf/2501.03262)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/reinforceplusplus_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | +| GSPO [[论文](https://arxiv.org/pdf/2507.18071)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/gspo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | | TOPR [[论文](https://arxiv.org/pdf/2503.14286)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/topr_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/topr_policy_loss.py)] | `algorithm_type: topr` | | sPPO [[论文](https://arxiv.org/pdf/2108.05828)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sppo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sppo_loss_fn.py)] | `algorithm_type: sppo` | | AsymRE [[论文](https://arxiv.org/pdf/2506.20520)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/asymre_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/asymre_advantage.py)] | `algorithm_type: asymre` | -| CISPO [[论文](https://arxiv.org/pdf/2506.13585)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | -| SAPO [[论文](https://arxiv.org/pdf/2511.20347)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | +| CISPO [[论文](https://arxiv.org/pdf/2506.13585)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/cispo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | +| SAPO [[论文](https://arxiv.org/pdf/2511.20347)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sapo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | | On-Policy Distillation [[博客](https://thinkingmachines.ai/blog/on-policy-distillation/)] [[论文](https://arxiv.org/pdf/2306.13649)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/common/workflows/on_policy_distill_workflow.py)] | `algorithm_type: on_policy_distill` | | JSD（Jensen-Shannon 散度） | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k/opd_gsm8k_jsd.yaml)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/jsd_advantage.py)] | `algorithm_type: jsd` | diff --git a/docs/sphinx_doc/source/main.md b/docs/sphinx_doc/source/main.md index 066fbbc0f42..234a419af16 100644 --- a/docs/sphinx_doc/source/main.md +++ b/docs/sphinx_doc/source/main.md @@ -80,14 +80,14 @@ We list some algorithms supported by Trinity-RFT in the following table. For mor | DPO [[Paper](https://arxiv.org/pdf/2305.18290)] | [[HumanLike Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/dpo_humanlike)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/dpo_loss.py)] | `algorithm_type: dpo` | | CHORD 💡 [[Paper](https://arxiv.org/pdf/2508.11408)] | [[Doc](https://agentscope-ai.github.io/Trinity-RFT/en/main/tutorial/example_mix_algo.html)] [[ToolACE Example](https://github.com/agentscope-ai/Trinity-RFT/blob/main/examples/mix_chord/mix_chord_toolace.yaml)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/chord_policy_loss.py)] | `algorithm_type: mix_chord` | | REC Series 💡 [[Paper](https://arxiv.org/pdf/2509.24203)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rec_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/rec_policy_loss.py)] | `algorithm_type: rec` | -| RLOO [[Paper](https://arxiv.org/pdf/2402.14740)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | -| REINFORCE++ [[Paper](https://arxiv.org/pdf/2501.03262)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | -| GSPO [[Paper](https://arxiv.org/pdf/2507.18071)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | +| RLOO [[Paper](https://arxiv.org/pdf/2402.14740)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rloo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | +| REINFORCE++ [[Paper](https://arxiv.org/pdf/2501.03262)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/reinforceplusplus_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | +| GSPO [[Paper](https://arxiv.org/pdf/2507.18071)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/gspo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | | TOPR [[Paper](https://arxiv.org/pdf/2503.14286)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/topr_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/topr_policy_loss.py)] | `algorithm_type: topr` | | sPPO [[Paper](https://arxiv.org/pdf/2108.05828)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sppo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sppo_loss_fn.py)] | `algorithm_type: sppo` | | AsymRE [[Paper](https://arxiv.org/pdf/2506.20520)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/asymre_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/asymre_advantage.py)] | `algorithm_type: asymre` | -| CISPO [[Paper](https://arxiv.org/pdf/2506.13585)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | -| SAPO [[Paper](https://arxiv.org/pdf/2511.20347)] | - | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | +| CISPO [[Paper](https://arxiv.org/pdf/2506.13585)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/cispo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | +| SAPO [[Paper](https://arxiv.org/pdf/2511.20347)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sapo_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | | On-Policy Distillation [[Blog](https://thinkingmachines.ai/blog/on-policy-distillation/)] [[Paper](https://arxiv.org/pdf/2306.13649)] | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/common/workflows/on_policy_distill_workflow.py)] | `algorithm_type: on_policy_distill` | | JSD (Jensen-Shannon Divergence) | [[GSM8K Example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k/opd_gsm8k_jsd.yaml)] | [[Code](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/jsd_advantage.py)] | `algorithm_type: jsd` | diff --git a/docs/sphinx_doc/source/tutorial/example_dataset_perspective.md b/docs/sphinx_doc/source/tutorial/example_dataset_perspective.md index 786fa230479..5e4f9fa99f3 100644 --- a/docs/sphinx_doc/source/tutorial/example_dataset_perspective.md +++ b/docs/sphinx_doc/source/tutorial/example_dataset_perspective.md @@ -11,6 +11,10 @@ This guide provides an example list from the dataset perspective, where you can | | Multi-Step GRPO | AgentScope ReAct agent training | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/agentscope_react), [doc](https://agentscope-ai.github.io/Trinity-RFT/en/main/tutorial/example_react.html) | | | AsymRE | Regular RFT | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/asymre_gsm8k) | | | CISPO | Regular RFT | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/cispo_gsm8k) | +| | RLOO | Regular RFT | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rloo_gsm8k) | +| | REINFORCE++ | Regular RFT | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/reinforceplusplus_gsm8k) | +| | GSPO | Regular RFT | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/gspo_gsm8k) | +| | SAPO | Regular RFT | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sapo_gsm8k) | | | GRPO | Training with prioritized tasks | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/grpo_gsm8k_task_pipeline), [doc](https://agentscope-ai.github.io/Trinity-RFT/en/main/tutorial/example_data_functionalities.html#example-data-processor-for-task-pipeline) | | | GRPO | Training with reward reshaping on experiences | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/grpo_gsm8k_experience_pipeline), [doc](https://agentscope-ai.github.io/Trinity-RFT/en/main/tutorial/example_data_functionalities.html#example-data-processor-for-experience-pipeline) | | | GRPO | Training with RULER (Relative Universal LLM-Elicited Rewards) | [example](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/grpo_gsm8k_ruler) | diff --git a/docs/sphinx_doc/source_zh/main.md b/docs/sphinx_doc/source_zh/main.md index 9ed175386c6..8744000b855 100644 --- a/docs/sphinx_doc/source_zh/main.md +++ b/docs/sphinx_doc/source_zh/main.md @@ -75,14 +75,14 @@ Trinity-RFT 面向不同背景和目标的用户提供相应功能： | DPO [[论文](https://arxiv.org/pdf/2305.18290)] | [[HumanLike 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/dpo_humanlike)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/dpo_loss.py)] | `algorithm_type: dpo` | | CHORD 💡 [[论文](https://arxiv.org/pdf/2508.11408)] | [[文档](https://agentscope-ai.github.io/Trinity-RFT/zh/main/tutorial/example_mix_algo.html)] [[ToolACE 示例](https://github.com/agentscope-ai/Trinity-RFT/blob/main/examples/mix_chord/mix_chord_toolace.yaml)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/chord_policy_loss.py)] | `algorithm_type: mix_chord` | | REC Series 💡 [[论文](https://arxiv.org/pdf/2509.24203)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rec_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/rec_policy_loss.py)] | `algorithm_type: rec` | -| RLOO [[论文](https://arxiv.org/pdf/2402.14740)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | -| REINFORCE++ [[论文](https://arxiv.org/pdf/2501.03262)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | -| GSPO [[论文](https://arxiv.org/pdf/2507.18071)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | +| RLOO [[论文](https://arxiv.org/pdf/2402.14740)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rloo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/rloo_advantage.py)] | `algorithm_type: rloo` | +| REINFORCE++ [[论文](https://arxiv.org/pdf/2501.03262)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/reinforceplusplus_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/reinforce_advantage.py)] | `algorithm_type: reinforceplusplus` | +| GSPO [[论文](https://arxiv.org/pdf/2507.18071)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/gspo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/gspo_policy_loss.py)] | `algorithm_type: gspo` | | TOPR [[论文](https://arxiv.org/pdf/2503.14286)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/topr_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/topr_policy_loss.py)] | `algorithm_type: topr` | | sPPO [[论文](https://arxiv.org/pdf/2108.05828)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sppo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sppo_loss_fn.py)] | `algorithm_type: sppo` | | AsymRE [[论文](https://arxiv.org/pdf/2506.20520)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/asymre_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/asymre_advantage.py)] | `algorithm_type: asymre` | -| CISPO [[论文](https://arxiv.org/pdf/2506.13585)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | -| SAPO [[论文](https://arxiv.org/pdf/2511.20347)] | - | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | +| CISPO [[论文](https://arxiv.org/pdf/2506.13585)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/cispo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/cispo_policy_loss.py)] | `algorithm_type: cispo` | +| SAPO [[论文](https://arxiv.org/pdf/2511.20347)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sapo_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/policy_loss_fn/sapo_policy_loss.py)] | `algorithm_type: sapo` | | On-Policy Distillation [[博客](https://thinkingmachines.ai/blog/on-policy-distillation/)] [[论文](https://arxiv.org/pdf/2306.13649)] | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/common/workflows/on_policy_distill_workflow.py)] | `algorithm_type: on_policy_distill` | | JSD（Jensen-Shannon 散度） | [[GSM8K 示例](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/opd_gsm8k/opd_gsm8k_jsd.yaml)] | [[代码](https://github.com/agentscope-ai/Trinity-RFT/tree/main/trinity/algorithm/advantage_fn/jsd_advantage.py)] | `algorithm_type: jsd` | diff --git a/docs/sphinx_doc/source_zh/tutorial/example_dataset_perspective.md b/docs/sphinx_doc/source_zh/tutorial/example_dataset_perspective.md index 07a42551c59..6ba3d0a3759 100644 --- a/docs/sphinx_doc/source_zh/tutorial/example_dataset_perspective.md +++ b/docs/sphinx_doc/source_zh/tutorial/example_dataset_perspective.md @@ -11,6 +11,10 @@ | | Multi-Step GRPO | AgentScope ReAct 智能体训练 | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/agentscope_react), [相关文档](https://agentscope-ai.github.io/Trinity-RFT/zh/main/tutorial/example_react.html) | | | AsymRE | 常规 RFT | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/asymre_gsm8k) | | | CISPO | 常规 RFT | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/cispo_gsm8k) | +| | RLOO | 常规 RFT | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/rloo_gsm8k) | +| | REINFORCE++ | 常规 RFT | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/reinforceplusplus_gsm8k) | +| | GSPO | 常规 RFT | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/gspo_gsm8k) | +| | SAPO | 常规 RFT | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/sapo_gsm8k) | | | GRPO | 使用优先级任务进行训练 | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/grpo_gsm8k_task_pipeline), [相关文档](https://agentscope-ai.github.io/Trinity-RFT/en/main/tutorial/example_data_functionalities.html#example-data-processor-for-task-pipeline) | | | GRPO | 在经验上进行奖励重塑的训练 | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/grpo_gsm8k_experience_pipeline), [相关文档](https://agentscope-ai.github.io/Trinity-RFT/zh/main/tutorial/example_data_functionalities.html#example-data-processor-for-experience-pipeline) | | | GRPO | 使用 RULER (Relative Universal LLM-Elicited Rewards) 进行训练 | [样例位置](https://github.com/agentscope-ai/Trinity-RFT/tree/main/examples/grpo_gsm8k_ruler) | diff --git a/examples/gspo_gsm8k/README.md b/examples/gspo_gsm8k/README.md new file mode 100644 index 00000000000..4cc06196d50 --- /dev/null +++ b/examples/gspo_gsm8k/README.md @@ -0,0 +1,5 @@ +# GSPO on GSM8K dataset + +This example shows the usage of [GSPO](https://arxiv.org/pdf/2507.18071) on the GSM8K dataset. + +The config file is located in [`gsm8k.yaml`](gsm8k.yaml). diff --git a/examples/gspo_gsm8k/gsm8k.yaml b/examples/gspo_gsm8k/gsm8k.yaml new file mode 100644 index 00000000000..3dc64c1aba4 --- /dev/null +++ b/examples/gspo_gsm8k/gsm8k.yaml @@ -0,0 +1,83 @@ +project: "Trinity-RFT-gsm8k" +name: "qwen2.5-1.5B-gsm8k-gspo" +checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints} +algorithm: + algorithm_type: gspo + repeat_times: 8 + optimizer: + lr: 1e-5 +model: + model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct} + max_response_tokens: 1024 + max_model_len: 2048 +cluster: + node_num: 1 + gpu_per_node: 2 +buffer: + total_epochs: 1 + batch_size: 96 + explorer_input: + taskset: + name: gsm8k + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'train' + format: + prompt_key: 'question' + response_key: 'answer' + rollout_args: + temperature: 1.0 + eval_tasksets: + - name: gsm8k-eval + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'test' + format: + prompt_key: 'question' + response_key: 'answer' + default_workflow_type: 'math_workflow' + trainer_input: + experience_buffer: + name: gsm8k_buffer + storage_type: queue + path: 'sqlite:///gsm8k.db' +explorer: + eval_interval: 50 + runner_per_model: 8 + rollout_model: + engine_num: 1 + tensor_parallel_size: 1 + enable_prefix_caching: false + enforce_eager: true + dtype: bfloat16 + seed: 42 +synchronizer: + sync_method: 'nccl' + sync_interval: 1 + sync_timeout: 1200 +trainer: + trainer_type: 'verl' + save_interval: 100 + grad_clip: 1.0 + use_dynamic_bsz: true + max_token_len_per_gpu: 16384 + ulysses_sequence_parallel_size: 1 +# stages: # Uncomment to add a SFT warmup stage before RFT +# - stage_name: sft_warmup +# mode: train +# algorithm: +# algorithm_type: sft +# buffer: +# train_batch_size: 128 +# total_steps: 10 +# trainer_input: +# experience_buffer: +# name: sft_warmup_dataset +# storage_type: file +# path: ${oc.env:TRINITY_SFT_DATASET_PATH} +# format: +# prompt_type: messages +# messages_key: 'messages' +# - stage_name: rft # leave empty to use the original configs for RFT diff --git a/examples/reinforceplusplus_gsm8k/README.md b/examples/reinforceplusplus_gsm8k/README.md new file mode 100644 index 00000000000..f5c097c31c6 --- /dev/null +++ b/examples/reinforceplusplus_gsm8k/README.md @@ -0,0 +1,5 @@ +# REINFORCE++ on GSM8K dataset + +This example shows the usage of [REINFORCE++](https://arxiv.org/pdf/2501.03262) on the GSM8K dataset. + +The config file is located in [`gsm8k.yaml`](gsm8k.yaml). diff --git a/examples/reinforceplusplus_gsm8k/gsm8k.yaml b/examples/reinforceplusplus_gsm8k/gsm8k.yaml new file mode 100644 index 00000000000..3c6757a352a --- /dev/null +++ b/examples/reinforceplusplus_gsm8k/gsm8k.yaml @@ -0,0 +1,83 @@ +project: "Trinity-RFT-gsm8k" +name: "qwen2.5-1.5B-gsm8k-reinforceplusplus" +checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints} +algorithm: + algorithm_type: reinforceplusplus + repeat_times: 8 + optimizer: + lr: 1e-5 +model: + model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct} + max_response_tokens: 1024 + max_model_len: 2048 +cluster: + node_num: 1 + gpu_per_node: 2 +buffer: + total_epochs: 1 + batch_size: 96 + explorer_input: + taskset: + name: gsm8k + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'train' + format: + prompt_key: 'question' + response_key: 'answer' + rollout_args: + temperature: 1.0 + eval_tasksets: + - name: gsm8k-eval + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'test' + format: + prompt_key: 'question' + response_key: 'answer' + default_workflow_type: 'math_workflow' + trainer_input: + experience_buffer: + name: gsm8k_buffer + storage_type: queue + path: 'sqlite:///gsm8k.db' +explorer: + eval_interval: 50 + runner_per_model: 8 + rollout_model: + engine_num: 1 + tensor_parallel_size: 1 + enable_prefix_caching: false + enforce_eager: true + dtype: bfloat16 + seed: 42 +synchronizer: + sync_method: 'nccl' + sync_interval: 1 + sync_timeout: 1200 +trainer: + trainer_type: 'verl' + save_interval: 100 + grad_clip: 1.0 + use_dynamic_bsz: true + max_token_len_per_gpu: 16384 + ulysses_sequence_parallel_size: 1 +# stages: # Uncomment to add a SFT warmup stage before RFT +# - stage_name: sft_warmup +# mode: train +# algorithm: +# algorithm_type: sft +# buffer: +# train_batch_size: 128 +# total_steps: 10 +# trainer_input: +# experience_buffer: +# name: sft_warmup_dataset +# storage_type: file +# path: ${oc.env:TRINITY_SFT_DATASET_PATH} +# format: +# prompt_type: messages +# messages_key: 'messages' +# - stage_name: rft # leave empty to use the original configs for RFT diff --git a/examples/rloo_gsm8k/README.md b/examples/rloo_gsm8k/README.md new file mode 100644 index 00000000000..7798d5dc0d6 --- /dev/null +++ b/examples/rloo_gsm8k/README.md @@ -0,0 +1,5 @@ +# RLOO on GSM8K dataset + +This example shows the usage of [RLOO](https://arxiv.org/pdf/2402.14740) on the GSM8K dataset. + +The config file is located in [`gsm8k.yaml`](gsm8k.yaml). diff --git a/examples/rloo_gsm8k/gsm8k.yaml b/examples/rloo_gsm8k/gsm8k.yaml new file mode 100644 index 00000000000..aa8589dc27e --- /dev/null +++ b/examples/rloo_gsm8k/gsm8k.yaml @@ -0,0 +1,83 @@ +project: "Trinity-RFT-gsm8k" +name: "qwen2.5-1.5B-gsm8k-rloo" +checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints} +algorithm: + algorithm_type: rloo + repeat_times: 8 + optimizer: + lr: 1e-5 +model: + model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct} + max_response_tokens: 1024 + max_model_len: 2048 +cluster: + node_num: 1 + gpu_per_node: 2 +buffer: + total_epochs: 1 + batch_size: 96 + explorer_input: + taskset: + name: gsm8k + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'train' + format: + prompt_key: 'question' + response_key: 'answer' + rollout_args: + temperature: 1.0 + eval_tasksets: + - name: gsm8k-eval + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'test' + format: + prompt_key: 'question' + response_key: 'answer' + default_workflow_type: 'math_workflow' + trainer_input: + experience_buffer: + name: gsm8k_buffer + storage_type: queue + path: 'sqlite:///gsm8k.db' +explorer: + eval_interval: 50 + runner_per_model: 8 + rollout_model: + engine_num: 1 + tensor_parallel_size: 1 + enable_prefix_caching: false + enforce_eager: true + dtype: bfloat16 + seed: 42 +synchronizer: + sync_method: 'nccl' + sync_interval: 1 + sync_timeout: 1200 +trainer: + trainer_type: 'verl' + save_interval: 100 + grad_clip: 1.0 + use_dynamic_bsz: true + max_token_len_per_gpu: 16384 + ulysses_sequence_parallel_size: 1 +# stages: # Uncomment to add a SFT warmup stage before RFT +# - stage_name: sft_warmup +# mode: train +# algorithm: +# algorithm_type: sft +# buffer: +# train_batch_size: 128 +# total_steps: 10 +# trainer_input: +# experience_buffer: +# name: sft_warmup_dataset +# storage_type: file +# path: ${oc.env:TRINITY_SFT_DATASET_PATH} +# format: +# prompt_type: messages +# messages_key: 'messages' +# - stage_name: rft # leave empty to use the original configs for RFT diff --git a/examples/sapo_gsm8k/README.md b/examples/sapo_gsm8k/README.md new file mode 100644 index 00000000000..9e25978880f --- /dev/null +++ b/examples/sapo_gsm8k/README.md @@ -0,0 +1,5 @@ +# SAPO on GSM8K dataset + +This example shows the usage of [SAPO](https://arxiv.org/pdf/2511.20347) on the GSM8K dataset. + +The config file is located in [`gsm8k.yaml`](gsm8k.yaml). diff --git a/examples/sapo_gsm8k/gsm8k.yaml b/examples/sapo_gsm8k/gsm8k.yaml new file mode 100644 index 00000000000..9e8ec9074f4 --- /dev/null +++ b/examples/sapo_gsm8k/gsm8k.yaml @@ -0,0 +1,83 @@ +project: "Trinity-RFT-gsm8k" +name: "qwen2.5-1.5B-gsm8k-sapo" +checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints} +algorithm: + algorithm_type: sapo + repeat_times: 8 + optimizer: + lr: 1e-5 +model: + model_path: ${oc.env:TRINITY_MODEL_PATH,Qwen/Qwen2.5-1.5B-Instruct} + max_response_tokens: 1024 + max_model_len: 2048 +cluster: + node_num: 1 + gpu_per_node: 2 +buffer: + total_epochs: 1 + batch_size: 96 + explorer_input: + taskset: + name: gsm8k + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'train' + format: + prompt_key: 'question' + response_key: 'answer' + rollout_args: + temperature: 1.0 + eval_tasksets: + - name: gsm8k-eval + storage_type: file + path: ${oc.env:TRINITY_TASKSET_PATH,openai/gsm8k} + subset_name: 'main' + split: 'test' + format: + prompt_key: 'question' + response_key: 'answer' + default_workflow_type: 'math_workflow' + trainer_input: + experience_buffer: + name: gsm8k_buffer + storage_type: queue + path: 'sqlite:///gsm8k.db' +explorer: + eval_interval: 50 + runner_per_model: 8 + rollout_model: + engine_num: 1 + tensor_parallel_size: 1 + enable_prefix_caching: false + enforce_eager: true + dtype: bfloat16 + seed: 42 +synchronizer: + sync_method: 'nccl' + sync_interval: 1 + sync_timeout: 1200 +trainer: + trainer_type: 'verl' + save_interval: 100 + grad_clip: 1.0 + use_dynamic_bsz: true + max_token_len_per_gpu: 16384 + ulysses_sequence_parallel_size: 1 +# stages: # Uncomment to add a SFT warmup stage before RFT +# - stage_name: sft_warmup +# mode: train +# algorithm: +# algorithm_type: sft +# buffer: +# train_batch_size: 128 +# total_steps: 10 +# trainer_input: +# experience_buffer: +# name: sft_warmup_dataset +# storage_type: file +# path: ${oc.env:TRINITY_SFT_DATASET_PATH} +# format: +# prompt_type: messages +# messages_key: 'messages' +# - stage_name: rft # leave empty to use the original configs for RFT