From 787ed71b39b8705bc8e559b5f2784d399a59a9c7 Mon Sep 17 00:00:00 2001 From: WEN Hao Date: Tue, 27 Oct 2026 08:53:54 +0800 Subject: [PATCH] docs: add the first two task-oriented tutorials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tutorial-first-experiment.rst: compare FedAvg/FedProx/SCAFFOLD at two sampling ratios on synthetic data, then dissect what num_communications counts (messages, not rounds, with the discriminating check and the per-direction decomposition) and how communication-skipping algorithms (FedPD) change the picture; the shown code is embedded via literalinclude from the runnable script test-files/tutorial_first_experiment.py - tutorial-cli-matrix.rst: run an experiment matrix from a single YAML file; the demo config example-configs/fedsynthetic-demo.yml expands into four synthetic-data runs, and the page documents the per-arm tag that keeps same-minute runs from overwriting each other's logs - index.rst: new Tutorials toctree section; quickstart next steps now point at the tutorials - test/test_tutorial_scripts.py: behavioral smoke tests extended to both tutorials — the comparison-table invariants (90/180 communications), the FedPD coda (fewer communications than FedAvg with at least one real uplink), and the CLI matrix expansion (distinct tags, one log pair per arm, budget asserted from config.json) - zh_CN translations for the new and updated pages --- docs/source/index.rst | 7 + docs/source/locale/zh_CN/LC_MESSAGES/index.po | 34 ++- .../locale/zh_CN/LC_MESSAGES/quickstart.po | 79 +++-- .../zh_CN/LC_MESSAGES/tutorial-cli-matrix.po | 176 +++++++++++ .../LC_MESSAGES/tutorial-first-experiment.po | 274 ++++++++++++++++++ docs/source/quickstart.rst | 9 +- docs/source/tutorial-cli-matrix.rst | 86 ++++++ docs/source/tutorial-first-experiment.rst | 158 ++++++++++ example-configs/fedsynthetic-demo.yml | 40 +++ test-files/tutorial_first_experiment.py | 117 ++++++++ test/test_tutorial_scripts.py | 88 +++++- 11 files changed, 1003 insertions(+), 65 deletions(-) create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/tutorial-cli-matrix.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/tutorial-first-experiment.po create mode 100644 docs/source/tutorial-cli-matrix.rst create mode 100644 docs/source/tutorial-first-experiment.rst create mode 100644 example-configs/fedsynthetic-demo.yml create mode 100644 test-files/tutorial_first_experiment.py diff --git a/docs/source/index.rst b/docs/source/index.rst index ce2517b2..bdb96ede 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,13 @@ Project Links: `GitHub `_ | `gitee \n" -"Language-Team: zh \n" "Language: zh_CN\n" +"Language-Team: zh \n" +"Plural-Forms: nplurals=1; plural=0;\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Plural-Forms: nplurals=1; plural=0;\n" -"Generated-By: Babel 2.12.1\n" +"Generated-By: Babel 2.16.0\n" #: ../../source/index.rst:13 msgid "Getting started" msgstr "快速开始" -#: ../../source/index.rst:20 +#: ../../source/index.rst:21 +msgid "Tutorials" +msgstr "教程" + +#: ../../source/index.rst:28 msgid "API Reference" msgstr "API 参考" -#: ../../source/index.rst:31 +#: ../../source/index.rst:41 msgid "Advanced Topics" msgstr "进阶主题" @@ -40,21 +44,25 @@ msgid "A Simple Simulation Framework for Federated Learning Based on PyTorch." msgstr "一个基于 PyTorch 的简单联邦学习仿真框架。" #: ../../source/index.rst:11 -msgid "Project Links: `GitHub `_ | `gitee `_" -msgstr "项目链接： `GitHub `_ | `gitee `_" +msgid "" +"Project Links: `GitHub `_ | `gitee " +"`_" +msgstr "" +"项目链接： `GitHub `_ | `gitee " +"`_" -#: ../../source/index.rst:40 +#: ../../source/index.rst:51 msgid "Indices and tables" msgstr "索引和表格" -#: ../../source/index.rst:42 +#: ../../source/index.rst:53 msgid ":ref:`genindex`" msgstr "" -#: ../../source/index.rst:43 +#: ../../source/index.rst:54 msgid ":ref:`modindex`" msgstr "" -#: ../../source/index.rst:44 +#: ../../source/index.rst:55 msgid ":ref:`search`" msgstr "" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/quickstart.po b/docs/source/locale/zh_CN/LC_MESSAGES/quickstart.po index 81d00924..7ca7b991 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/quickstart.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/quickstart.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: fl-sim \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-10-26 13:08+0800\n" +"POT-Creation-Date: 2026-10-27 08:46+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: zh_CN\n" @@ -31,8 +31,8 @@ msgid "" "takes seconds on a laptop — no dataset download, no GPU required." msgstr "" "本页面带你完成第一次联邦训练：安装 fl-sim，在合成数据集上训练 " -":class:`~fl_sim.algorithms.fedopt.FedAvg`，并查看产出的日志。整个过程在笔记本电脑上" -"几秒即可完成——无需下载数据集，也无需 GPU。" +":class:`~fl_sim.algorithms.fedopt.FedAvg`，并查看产出的日志。整个过程在笔记本电脑上几秒即可完成——无需下载数据集，也无需" +" GPU。" #: ../../source/quickstart.rst:11 msgid "" @@ -50,8 +50,8 @@ msgid "" "git+https://github.com/wenh06/fl-sim.git``). Python 3.10+ with PyTorch is" " all you need." msgstr "" -"按照 :doc:`install` 安装 fl-sim（``pip install " -"git+https://github.com/wenh06/fl-sim.git``）。只需 Python 3.10+ 与 PyTorch。" +"按照 :doc:`install` 安装 fl-sim（``pip install git+https://github.com/wenh06" +"/fl-sim.git``）。只需 Python 3.10+ 与 PyTorch。" #: ../../source/quickstart.rst:21 msgid "Train your first model" @@ -67,10 +67,9 @@ msgid "" "client count, sampling ratio); the client config carries the local " "training recipe." msgstr "" -"fl-sim 中的每次联邦训练都由三个对象构成： **数据集** （数据本身以及在客户端之间的" -"划分方式）、 **模型** （一个 PyTorch 模块，这里取自数据集内置的候选模型）和" -" **服务器** （持有全局模型，负责采样客户端与聚合）。服务器配置承载联邦训练的调度" -"（轮数、客户端数量、采样比例）；客户端配置承载本地训练方案。" +"fl-sim 中的每次联邦训练都由三个对象构成： **数据集** （数据本身以及在客户端之间的划分方式）、 **模型** （一个 PyTorch " +"模块，这里取自数据集内置的候选模型）和 **服务器** " +"（持有全局模型，负责采样客户端与聚合）。服务器配置承载联邦训练的调度（轮数、客户端数量、采样比例）；客户端配置承载本地训练方案。" #: ../../source/quickstart.rst:35 msgid "" @@ -82,11 +81,10 @@ msgid "" "full federated loop; you will see per-round progress and metrics in the " "console." msgstr "" -"把这段代码粘贴到脚本（或 REPL）里运行。``FedSynthetic`` 会在本地生成一个划分给 " -"30 个客户端的合成分类数据集——不下载任何东西。" -"``FedAvgServerConfig(num_iters=5, num_clients=10, clients_sample_ratio=1.0)`` " -"运行 **5 轮**，每轮从 30 个客户端中采样 **10 个**，且全部参与。" -"``server.train()`` 运行完整的联邦循环；你会在控制台看到每轮的进度与指标。" +"把这段代码粘贴到脚本（或 REPL）里运行。``FedSynthetic`` 会在本地生成一个划分给 30 " +"个客户端的合成分类数据集——不下载任何东西。``FedAvgServerConfig(num_iters=5, num_clients=10, " +"clients_sample_ratio=1.0)`` 运行 **5 轮**，每轮从 30 个客户端中采样 **10 " +"个**，且全部参与。``server.train()`` 运行完整的联邦循环；你会在控制台看到每轮的进度与指标。" #: ../../source/quickstart.rst:44 msgid "" @@ -95,7 +93,8 @@ msgid "" " — see the :meth:`~fl_sim.nodes.Server.train` API." msgstr "" "``server.train()`` 即 ``server.train(\"federated\")``。fl-sim 也可以用同样的对象运行 " -"``\"centralized\"`` 与 ``\"local\"`` 基线——参见 :meth:`~fl_sim.nodes.Server.train` API。" +"``\"centralized\"`` 与 ``\"local\"`` 基线——参见 " +":meth:`~fl_sim.nodes.Server.train` API。" #: ../../source/quickstart.rst:49 msgid "Where do the logs go?" @@ -107,8 +106,8 @@ msgid "" "subdirectory with ``log_dir=`` in the server config, as done here). The " "script above reads the metrics back and prints the result:" msgstr "" -"每次运行都会把日志写入 ``~/.cache/fl-sim/.logs/`` （可通过服务器配置里的 " -"``log_dir=`` 指定子目录，如本例所示）。上面的脚本会把指标读回来并打印结果：" +"每次运行都会把日志写入 ``~/.cache/fl-sim/.logs/`` （可通过服务器配置里的 ``log_dir=`` " +"指定子目录，如本例所示）。上面的脚本会把指标读回来并打印结果：" #: ../../source/quickstart.rst:62 msgid "Three files per run:" @@ -125,17 +124,15 @@ msgid "" "...}}`` — one dict per evaluation, holding ``step``, ``acc``, ``loss``, " "etc. This is the file you will plot when visualizing results;" msgstr "" -"一个 ``.json`` 日志，保存完整的指标历史，结构为 " -"``{\"train\": {...}, \"val\": {\"Server\": [...], \"Client0\": [...], " -"...}}``——每次评估对应一个字典，包含 ``step``、``acc``、``loss`` 等字段。" -"可视化结果时用的就是这个文件；" +"一个 ``.json`` 日志，保存完整的指标历史，结构为 ``{\"train\": {...}, \"val\": {\"Server\": " +"[...], \"Client0\": [...], ...}}``——每次评估对应一个字典，包含 " +"``step``、``acc``、``loss`` 等字段。可视化结果时用的就是这个文件；" #: ../../source/quickstart.rst:69 msgid "" "``config.json`` — the exact server and client configs of the run, so " "every experiment can be reproduced later." -msgstr "" -"``config.json``——本次运行的确切服务器与客户端配置，便于日后复现每个实验。" +msgstr "``config.json``——本次运行的确切服务器与客户端配置，便于日后复现每个实验。" #: ../../source/quickstart.rst:74 msgid "" @@ -145,39 +142,41 @@ msgid "" "json snapshot is taken at the start of each round, so the final " "communication count is higher than the last logged step." msgstr "" -"日志中的 ``step`` 统计的是**通信次数**，而不是轮数：每轮对每个被采样客户端包含一次" -"下行广播与一次上行回复（这里每轮采样 10 个客户端，因此 step 每轮前进 20）。json " -"中的快照记录于每轮开始时，因此最终通信次数会大于最后一条日志的 step。" +"日志中的 ``step`` 统计的是**通信次数**，而不是轮数：每轮对每个被采样客户端包含一次下行广播与一次上行回复（这里每轮采样 10 " +"个客户端，因此 step 每轮前进 20）。json 中的快照记录于每轮开始时，因此最终通信次数会大于最后一条日志的 step。" #: ../../source/quickstart.rst:80 msgid "" "(The exact accuracy value may differ across machines and package " -"versions; on one machine this run ends at 0.8767, up from 0.16 at " -"initialization.)" -msgstr "" -"（具体精度数值可能因机器与包版本而异；在一种机器上，该运行的最终精度为 0.8767，" -"初始时为 0.16。）" +"versions; on one machine this run ends at 0.8767, up from 0.011 — chance " +"level for 10 classes — at initialization.)" +msgstr "（具体精度数值可能因机器与包版本而异；在一种机器上，该运行的最终精度为 0.8767，初始时为 0.011——即 10 类任务的随机水平。）" -#: ../../source/quickstart.rst:84 +#: ../../source/quickstart.rst:85 msgid "Next steps" msgstr "下一步" -#: ../../source/quickstart.rst:86 +#: ../../source/quickstart.rst:87 msgid "" ":doc:`concepts` — what actually happens in each round: sampling, " "broadcast, local updates, aggregation." msgstr ":doc:`concepts`——每一轮实际发生了什么：采样、广播、本地更新、聚合。" -#: ../../source/quickstart.rst:88 +#: ../../source/quickstart.rst:89 msgid "" -":doc:`cli` — run experiments from the command line with YAML config files" -" (a full tutorial on experiment batches is on the way)." +":doc:`tutorial-first-experiment` — turn this single run into a controlled" +" comparison across algorithms and sampling ratios." msgstr "" -":doc:`cli`——用 YAML 配置文件从命令行运行实验（关于实验批次的完整教程也在准备中）。" +":doc:`tutorial-first-experiment`——把这个单次运行变成跨算法、跨采样比例的对照实验。" -#: ../../source/quickstart.rst:90 +#: ../../source/quickstart.rst:91 +msgid "" +":doc:`tutorial-cli-matrix` — run experiment batches from the CLI with " +"YAML config files." +msgstr ":doc:`tutorial-cli-matrix`——用 YAML 配置文件从命令行跑实验批次。" + +#: ../../source/quickstart.rst:93 msgid "" ":doc:`parallel_execution` — speed training up with a parallel executor " "backend while keeping results reproducible." -msgstr "" -":doc:`parallel_execution`——用并行执行后端加速训练，同时保持结果可复现。" +msgstr ":doc:`parallel_execution`——用并行执行后端加速训练，同时保持结果可复现。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/tutorial-cli-matrix.po b/docs/source/locale/zh_CN/LC_MESSAGES/tutorial-cli-matrix.po new file mode 100644 index 00000000..1573bbaf --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/tutorial-cli-matrix.po @@ -0,0 +1,176 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2023, WEN Hao +# This file is distributed under the same license as the fl-sim package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: fl-sim \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-10-27 08:46+0800\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.16.0\n" + +#: ../../source/tutorial-cli-matrix.rst:4 +msgid "Tutorial: experiment matrices from the CLI" +msgstr "教程：用 CLI 跑实验矩阵" + +#: ../../source/tutorial-cli-matrix.rst:6 +msgid "" +"Writing one Python script per experiment does not scale. fl-sim's command" +" line interface runs a whole **experiment matrix** from a single YAML " +"file: you describe the grid of configurations once, and the CLI expands " +"and executes every combination. This tutorial builds a four-run matrix on" +" synthetic data and shows where the results land." +msgstr "" +"每个实验写一个 Python 脚本是无法扩展的。fl-sim 的命令行接口能用单个 YAML 文件跑起" +"一整个 **实验矩阵** ：你把配置网格描述一次，CLI 负责展开并执行每个组合。本教程在" +"合成数据上构建一个四次运行的矩阵，并展示结果落在哪里。" + +#: ../../source/tutorial-cli-matrix.rst:12 +msgid "" +"The YAML syntax reference lives in :doc:`cli`; this page is the workflow " +"narrative. Prerequisites: :doc:`tutorial-first-experiment`." +msgstr "" +"YAML 语法参考见 :doc:`cli` ；本页讲的是工作流程。前置阅读：" +":doc:`tutorial-first-experiment` 。" + +#: ../../source/tutorial-cli-matrix.rst:16 +msgid "The matrix file" +msgstr "矩阵文件" + +#: ../../source/tutorial-cli-matrix.rst:18 +msgid "" +"Here is the complete config for this tutorial (``example-configs" +"/fedsynthetic-demo.yml``):" +msgstr "本教程的完整配置如下（``example-configs/fedsynthetic-demo.yml`` ）：" + +#: ../../source/tutorial-cli-matrix.rst:24 +msgid "Three sections matter:" +msgstr "三个关键部分：" + +#: ../../source/tutorial-cli-matrix.rst:26 +msgid "" +"``strategy.matrix`` — the grid. Every combination of the listed values is" +" run once: 2 sampling ratios × 2 seeds = 4 experiments." +msgstr "" +"``strategy.matrix``——网格。列出的值每个组合各跑一次：2 个采样比例 × 2 个种子" +"= 4 个实验。" + +#: ../../source/tutorial-cli-matrix.rst:28 +msgid "" +"``${{ matrix. }}`` — interpolation. Wherever this pattern appears, " +"the CLI substitutes the current combination's value. ``seed`` sits at the" +" top level, so it seeds the dataset *and* the server config of that run." +msgstr "" +"``${{ matrix. }}``——插值。该模式出现之处，CLI 都会代入当前组合的值。" +"``seed`` 位于顶层，因此它同时为该次运行的数据集 *和* 服务器配置设定种子。" + +#: ../../source/tutorial-cli-matrix.rst:32 +msgid "" +"everything else — ``algorithm`` (name plus server/client configs), " +"``dataset``, ``model`` — is exactly the config you would pass in Python, " +"per :doc:`quickstart`, just spelled as YAML. Dataset-specific constructor" +" arguments (``alpha``, ``beta``, ``iid``, ``num_clients`` for " +":class:`~fl_sim.data_processing.fed_synthetic.FedSynthetic`) are passed " +"straight through." +msgstr "" +"其余部分—— ``algorithm`` （名字加服务器/客户端配置）、 ``dataset`` 、 ``model`` ——" +"与你在 Python 里传入的配置（见 :doc:`quickstart` ）完全一致，只是改写成 YAML。" +"数据集各自的构造参数（ :class:`~fl_sim.data_processing.fed_synthetic.FedSynthetic` " +"的 ``alpha`` 、 ``beta`` 、 ``iid`` 、 ``num_clients`` ）原样透传。" + +#: ../../source/tutorial-cli-matrix.rst:40 +msgid "Run it" +msgstr "运行" + +#: ../../source/tutorial-cli-matrix.rst:46 +msgid "" +"You will see the four runs execute in sequence, each logging its " +"combination (``ratio=0.5 seed=0``, ...). Total time: seconds — no " +"downloads." +msgstr "" +"你会看到四次运行依次执行，每次都会记录自己的组合（``ratio=0.5 seed=0`` 等）。" +"总耗时：几秒——无下载。" + +#: ../../source/tutorial-cli-matrix.rst:51 +msgid "Where the results land" +msgstr "结果落在哪里" + +#: ../../source/tutorial-cli-matrix.rst:53 +msgid "" +"All runs write into ``~/.cache/fl-sim/.logs/fedsynthetic-demo/`` (the " +"``log_dir`` of this config). Per run you get one ``.txt`` and one " +"``.json`` log — the same files :doc:`quickstart` described — plus one " +"``config.json`` holding the exact server/client configs of the most " +"recent run (a previous ``config.json`` is rotated to " +"``config.json.bak``)." +msgstr "" +"所有运行都写入 ``~/.cache/fl-sim/.logs/fedsynthetic-demo/``（本配置的 ``log_dir`` ）。" +"每次运行各有一份 ``.txt`` 与一份 ``.json`` 日志——即 :doc:`quickstart` 描述过的" +"那些文件——另有一份 ``config.json`` 保存最近一次运行的确切服务器/客户端配置" +"（已有的 ``config.json`` 会被轮转为 ``config.json.bak`` ）。" + +#: ../../source/tutorial-cli-matrix.rst:60 +msgid "" +"One detail that matters as soon as runs finish within the same minute: " +"log file names carry a timestamp **plus the run's ``tag``**. Without a " +"distinct tag per matrix combination, same-named logs overwrite each " +"other. That is what the ``tag: ratio_${{ ... }}_seed_${{ ... }}`` line in" +" the config above is for — give every matrix arm its own tag." +msgstr "" +"一个在多次运行同一分钟内结束时就会生效的细节：日志文件名带有时间戳 **加上运行的 " +"``tag``** 。若每个矩阵组合没有独特的 tag，同名的日志会互相覆盖。上面配置里的 " +"``tag: ratio_${{ ... }}_seed_${{ ... }}`` 正是为此——给矩阵的每个分支一个专属 tag。" + +#: ../../source/tutorial-cli-matrix.rst:67 +msgid "Beyond serial execution" +msgstr "超越串行执行" + +#: ../../source/tutorial-cli-matrix.rst:69 +msgid "" +"A matrix of long experiments wants more than one process. The " +"``strategy`` section also accepts parallel options (``n_parallel``, or a " +"``parallel: {mode: parallel_task, num_workers: N}`` block) that dispatch " +"matrix arms to worker processes — see ``example-configs/multiprocessing-" +"demo.yml`` for a complete example, and :doc:`parallel_execution` for the " +"in-process executor backend that parallelizes *within* a single run. The " +"two are orthogonal: process-level parallelism across matrix arms, " +"executor-level parallelism across clients." +msgstr "" +"长实验组成的矩阵需要不止一个进程。 ``strategy`` 节还接受并行选项（``n_parallel``，" +"或 ``parallel: {mode: parallel_task, num_workers: N}`` 块），把矩阵分支派发给工作" +"进程——完整示例见 ``example-configs/multiprocessing-demo.yml``；进程内把 *单次运行* " +"的客户端并行化的执行器后端见 :doc:`parallel_execution` 。两者正交：进程级并行作用于" +"矩阵分支之间，执行器级并行作用于客户端之间。" + +#: ../../source/tutorial-cli-matrix.rst:79 +msgid "Next steps" +msgstr "下一步" + +#: ../../source/tutorial-cli-matrix.rst:81 +msgid ":doc:`cli` — the full YAML syntax reference and CLI options." +msgstr ":doc:`cli`——完整的 YAML 语法参考与 CLI 选项。" + +#: ../../source/tutorial-cli-matrix.rst:82 +msgid ":doc:`parallel_execution` — speed up each individual run." +msgstr ":doc:`parallel_execution`——加速每一次单独的运行。" + +#: ../../source/tutorial-cli-matrix.rst:83 +msgid "" +"How-to guides on custom algorithms, datasets and optimizers are on the " +"way; until then, ``test-files/custom_alg.py``, ``test-" +"files/custom_dataset.py`` and ``test-files/custom_optimizer.py`` are " +"complete, runnable examples." +msgstr "" +"自定义算法、数据集与优化器的 how-to 指南正在准备中；在那之前，" +"``test-files/custom_alg.py``、``test-files/custom_dataset.py`` 与 " +"``test-files/custom_optimizer.py`` 就是完整可运行的例子。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/tutorial-first-experiment.po b/docs/source/locale/zh_CN/LC_MESSAGES/tutorial-first-experiment.po new file mode 100644 index 00000000..e6514677 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/tutorial-first-experiment.po @@ -0,0 +1,274 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2023, WEN Hao +# This file is distributed under the same license as the fl-sim package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: fl-sim \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-10-27 08:46+0800\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.16.0\n" + +#: ../../source/tutorial-first-experiment.rst:4 +msgid "Tutorial: your first controlled experiment" +msgstr "教程：你的第一个对照实验" + +#: ../../source/tutorial-first-experiment.rst:6 +msgid "" +":doc:`Quickstart ` ran one training once. A real experiment " +"compares configurations under identical conditions and reads the results " +"back. In this tutorial you will compare three algorithms at two sampling " +"ratios, then dissect what \"communication cost\" actually means in fl-" +"sim." +msgstr "" +":doc:`Quickstart ` 只跑了一次训练。真正的实验要在相同条件下比较不同配置，" +"并把结果读回来。本教程将比较三个算法在两种采样比例下的表现，然后剖析 fl-sim 中" +"\"通信开销\"究竟指什么。" + +#: ../../source/tutorial-first-experiment.rst:11 +msgid "" +"Everything runs on " +":class:`~fl_sim.data_processing.fed_synthetic.FedSynthetic` — no " +"downloads, seconds per run. The full script shown below lives at ``test-" +"files/tutorial_first_experiment.py`` and is exercised in CI." +msgstr "" +"全部运行基于 :class:`~fl_sim.data_processing.fed_synthetic.FedSynthetic`——" +"无需下载，每次运行只需几秒。下文展示的完整脚本位于 " +"``test-files/tutorial_first_experiment.py``，并在 CI 中被执行。" + +#: ../../source/tutorial-first-experiment.rst:15 +msgid "" +"Prerequisites: :doc:`quickstart` (the three-object model: dataset, model," +" server) and :doc:`concepts` (what a round does)." +msgstr "" +"前置阅读： :doc:`quickstart` （三对象模型：数据集、模型、服务器）与 " +" :doc:`concepts` （一轮训练做了什么）。" + +#: ../../source/tutorial-first-experiment.rst:19 +msgid "Set up the comparison" +msgstr "搭建对照" + +#: ../../source/tutorial-first-experiment.rst:21 +msgid "" +"Six runs: **FedAvg** (plain averaging), **FedProx** (proximal-regularized" +" local objectives) and **SCAFFOLD** (control variates) — the classic " +"gradient, proximal and variance-reduction axes — each at **sampling " +"ratios 0.5 and 1.0**." +msgstr "" +"六次运行： **FedAvg** （朴素平均）、 **FedProx** （近端正则的本地目标）与 " +" **SCAFFOLD** （控制变量）——梯度、近端、方差缩减三条经典轴线——各自在 " +" **采样比例 0.5 与 1.0** 下运行。" + +#: ../../source/tutorial-first-experiment.rst:26 +msgid "" +"One deliberate choice: the dataset is constructed **once, before any " +"config or server**. Config constructors re-seed the global RNG as they " +"are built, so whatever was constructed last would otherwise shift the " +"sampling stream of everything after it. The order *dataset → config → " +"server* keeps every run reproducible:" +msgstr "" +"一个刻意的选择：数据集 **只构造一次，且先于任何配置或服务器** 。配置类的构造器在" +"构建时会重置全局 RNG，谁最后构造谁生效，其后的采样流都会被移动。按 " +"*数据集 → 配置 → 服务器* 的顺序构造，每次运行都可复现：" + +#: ../../source/tutorial-first-experiment.rst:37 +msgid "" +"``final_accuracy`` reads a run's JSON log — the same file format you saw " +"in :doc:`quickstart`. SCAFFOLD needs its global step size ``lr`` passed " +"explicitly (its config has no default for it); FedPD will need a " +"frequency parameter later. Both are per-algorithm server keywords, " +"collected in ``SERVER_KW``." +msgstr "" +"``final_accuracy`` 读取一次运行的 JSON 日志——与 :doc:`quickstart` 中相同的文件" +"格式。SCAFFOLD 需要显式传入全局步长 ``lr``（它的配置没有默认值）；FedPD 稍后也需要" +"一个频率参数。两者都是算法各自的服务器关键字，统一收在 ``SERVER_KW`` 里。" + +#: ../../source/tutorial-first-experiment.rst:44 +msgid "Run and read the results" +msgstr "运行并读取结果" + +#: ../../source/tutorial-first-experiment.rst:46 +msgid "" +"The comparison itself is a loop, and the results land in a " +":class:`pandas.DataFrame`:" +msgstr "对照本身就是一个循环，结果落入 :class:`pandas.DataFrame` ：" + +#: ../../source/tutorial-first-experiment.rst:54 +msgid "" +"Output on one machine (your numbers may differ slightly by package " +"version):" +msgstr "在一种机器上的输出（你的数字可能因包版本略有差异）：" + +#: ../../source/tutorial-first-experiment.rst:67 +msgid "Two honest observations about this table:" +msgstr "对这张表的两点如实观察：" + +#: ../../source/tutorial-first-experiment.rst:69 +msgid "" +"This synthetic task is *easy* — FedProx and SCAFFOLD reach the same " +"accuracy ceiling as FedAvg at full sampling. Their advantages show up on " +"harder, more heterogeneous tasks (see the :doc:`algorithm theory pages " +"` for when each helps)." +msgstr "" +"这个合成任务 *太容易* ——全采样下 FedProx 与 SCAFFOLD 达到了和 FedAvg 相同的精度" +"天花板。它们的优势要在更难、异质性更强的任务上才显现（何时用哪个，见 " +":doc:`算法理论页 ` ）。" + +#: ../../source/tutorial-first-experiment.rst:74 +msgid "" +"The accuracy story saturates; the **communication** story does not. Every" +" run at ratio 0.5 used exactly 90 communications and every run at ratio " +"1.0 used exactly 180 — regardless of the algorithm. That number deserves " +"a closer look, because it is easy to misread." +msgstr "" +"精度的故事饱和了， **通信** 的故事没有：比例 0.5 的每次运行都恰好用了 90 次通信，" +"比例 1.0 的都恰好 180 次——与算法无关。这个数字值得细看，因为它很容易被误读。" + +#: ../../source/tutorial-first-experiment.rst:80 +msgid "What ``num_communications`` counts" +msgstr "``num_communications`` 数的是什么" + +#: ../../source/tutorial-first-experiment.rst:82 +msgid "" +"``num_communications`` counts **messages, not rounds**: each round adds " +"one downlink broadcast *per sampled client* plus one uplink reply *per " +"sampled client*. With 30 clients at ratio 0.5, every round is 15 " +"downlinks + 15 uplinks = 30 messages, so 3 rounds = 90; at ratio 1.0 that" +" doubles to 180." +msgstr "" +"``num_communications`` 统计的是 **消息数，不是轮数** ：每轮对 *每个被采样客户端* " +"加一次下行广播，再加一次上行回复。30 个客户端、比例 0.5 时，每轮是 15 下行 + " +"15 上行 = 30 条消息，3 轮 = 90；比例 1.0 时翻倍为 180。" + +#: ../../source/tutorial-first-experiment.rst:88 +msgid "" +"The discriminating check: **3 rounds at full sampling and 6 rounds at " +"half sampling both land on 180.** The counter scales with ``2 × " +"sampled_clients × rounds``, not with rounds. Reading it as " +"\"communication rounds\" would make 90 look like \"90 rounds\" — off by a" +" factor of 30." +msgstr "" +"判别性的检验： **3 轮全采样与 6 轮半采样都恰好是 180。** 计数随 ``2 × 采样数 × 轮数`` " +"缩放，而不是随轮数。把它读成\"通信轮数\"，90 就会被当成\"90 轮\"——差了 30 倍。" + +#: ../../source/tutorial-first-experiment.rst:94 +msgid "The decomposition to keep in mind:" +msgstr "值得记住的分解式：" + +#: ../../source/tutorial-first-experiment.rst:96 +msgid "\\text{FedAvg} = 2 \\times \\text{sampled} \\times \\text{rounds}" +msgstr "\\text{FedAvg} = 2 \\times \\text{sampled} \\times \\text{rounds}" + +#: ../../source/tutorial-first-experiment.rst:100 +msgid "" +"(Note also that the ``step`` recorded in the JSON log is a snapshot taken" +" at the *start* of each round: it trails the final counter by the last " +"round's full cost — read the final count from the server, as the scripts " +"here do.)" +msgstr "" +"（另注：JSON 日志里记录的 ``step`` 是 *每轮开始时* 的快照：它比最终计数少最后一轮的" +"全部开销——最终值请从服务器读取，正如本页脚本的做法。）" + +#: ../../source/tutorial-first-experiment.rst:106 +msgid "Sampling is not communicating" +msgstr "被采样不等于在通信" + +#: ../../source/tutorial-first-experiment.rst:108 +msgid "" +"All three algorithms above communicate every round. fl-sim also ships " +"**communication-skipping** algorithms, and they break the ``2 × sampled ×" +" rounds`` identity in an instructive way. Run FedPD — which uploads only " +"every ``int(1/p)``-th round — for five rounds next to a five-round " +"FedAvg:" +msgstr "" +"上面三个算法每一轮都在通信。fl-sim 还内置 **可跳过通信** 的算法，它们会以很有启发性的" +"方式打破 ``2 × 采样数 × 轮数`` 的恒等式。让 FedPD（每 ``int(1/p)`` 轮才上传一次）" +"与五轮 FedAvg 并排各跑五轮：" + +#: ../../source/tutorial-first-experiment.rst:119 +msgid "Output on one machine:" +msgstr "在一种机器上的输出：" + +#: ../../source/tutorial-first-experiment.rst:126 +msgid "" +"The decomposition says it precisely: **the downlink is identical** (75 = " +"5 rounds × 15 clients — broadcasting to the sampled set is never " +"skipped); FedPD saves *uplinks only*. Five FedPD rounds cost the same 90 " +"messages as three FedAvg rounds — the same communication budget buys two " +"extra local iterations." +msgstr "" +"分解式把话说得非常准： **下行完全相同** （75 = 5 轮 × 15 客户端——对被采样集合的广播" +"从不跳过）；FedPD 省下的 *全是上行* 。五轮 FedPD 花费的 90 条消息与三轮 FedAvg 相同" +"——同样的通信预算换来了多两轮的本地迭代。" + +#: ../../source/tutorial-first-experiment.rst:132 +msgid "Three practical details when you run this yourself:" +msgstr "自己动手跑时的三个实用细节：" + +#: ../../source/tutorial-first-experiment.rst:134 +msgid "" +"``p`` must satisfy **0 < p < 1**. With p = 1 FedPD uploads every round " +"(degenerating to FedAvg); with p > 1 the upload frequency becomes zero " +"and the run crashes. The config exposes ``p`` but the derived upload " +"frequency ``int(1/p)`` always wins — passing a ``comm_freq`` yourself is " +"silently overwritten." +msgstr "" +"``p`` 必须满足 **0 < p < 1** 。p = 1 时 FedPD 每轮都上传（退化为 FedAvg）；p > 1 时" +"上传频率变为零，运行直接崩溃。配置暴露的是 ``p`` ，派生的上传频率 ``int(1/p)`` " +"始终说了算——自己传 ``comm_freq`` 会被静默覆写。" + +#: ../../source/tutorial-first-experiment.rst:139 +msgid "" +"**You will see RuntimeWarnings — they are expected, not errors.** The " +"server logs ``No message received from the clients...`` once per skipped " +"round (\"nobody uploaded this round, so the global model did not move\")," +" and each skipping client logs ``No parameters received from server...`` " +"(\"I was not given a new global model this round, so I keep working from " +"my current parameters\"). At tutorial scale the client-side message " +"appears once per (skipped client × skipped round), so it dominates the " +"console. Both are how FedPD is designed to behave." +msgstr "" +" **你会看到 RuntimeWarning——那是预期行为，不是错误。** 服务器对每个跳过的轮次记录 " +"``No message received from the clients...`` （\"这轮没人上传，全局模型没有动\"）；" +"每个跳过的客户端记录 ``No parameters received from server...`` （\"这轮没收到新的" +"全局模型，我继续用当前参数\"）。教程规模下，客户端侧的消息每个（跳过客户端 × 跳过轮次）" +"出现一次，会刷满控制台。两者都是 FedPD 的设计行为。" + +#: ../../source/tutorial-first-experiment.rst:147 +msgid "" +"The JSON ``step`` snapshot trails differently per algorithm: for the " +"five-round FedAvg it ends at 120 (the last round's 30 messages are " +"unlogged), for FedPD at 75 (its last round never uploads). The final " +"counts come from ``server._num_communications``." +msgstr "" +"JSON 的 ``step`` 快照落后多少因算法而异：五轮 FedAvg 终于 120（最后一轮的 30 条消息" +"未入日志），FedPD 终于 75（它的最后一轮从不上传）。最终计数请以 " +"``server._num_communications`` 为准。" + +#: ../../source/tutorial-first-experiment.rst:153 +msgid "Next steps" +msgstr "下一步" + +#: ../../source/tutorial-first-experiment.rst:155 +msgid "" +":doc:`tutorial-cli-matrix` — run the same kind of comparison from the " +"command line, as a YAML-driven experiment matrix." +msgstr "" +":doc:`tutorial-cli-matrix`——用 YAML 实验矩阵从命令行跑同样的对照。" + +#: ../../source/tutorial-first-experiment.rst:157 +msgid "" +":doc:`parallel_execution` — speed these runs up with a parallel executor " +"without changing the results." +msgstr "" +":doc:`parallel_execution`——用并行执行后端加速这些运行，且不改变结果。" diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 896e3610..6c95e653 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -78,14 +78,17 @@ Three files per run: communication count is higher than the last logged step. (The exact accuracy value may differ across machines and package versions; -on one machine this run ends at 0.8767, up from 0.16 at initialization.) +on one machine this run ends at 0.8767, up from 0.011 — chance level for +10 classes — at initialization.) Next steps ---------- - :doc:`concepts` — what actually happens in each round: sampling, broadcast, local updates, aggregation. -- :doc:`cli` — run experiments from the command line with YAML config - files (a full tutorial on experiment batches is on the way). +- :doc:`tutorial-first-experiment` — turn this single run into a + controlled comparison across algorithms and sampling ratios. +- :doc:`tutorial-cli-matrix` — run experiment batches from the CLI with + YAML config files. - :doc:`parallel_execution` — speed training up with a parallel executor backend while keeping results reproducible. diff --git a/docs/source/tutorial-cli-matrix.rst b/docs/source/tutorial-cli-matrix.rst new file mode 100644 index 00000000..316748a2 --- /dev/null +++ b/docs/source/tutorial-cli-matrix.rst @@ -0,0 +1,86 @@ +.. _tutorial-cli-matrix: + +Tutorial: experiment matrices from the CLI +========================================== + +Writing one Python script per experiment does not scale. fl-sim's command +line interface runs a whole **experiment matrix** from a single YAML file: +you describe the grid of configurations once, and the CLI expands and +executes every combination. This tutorial builds a four-run matrix on +synthetic data and shows where the results land. + +The YAML syntax reference lives in :doc:`cli`; this page is the workflow +narrative. Prerequisites: :doc:`tutorial-first-experiment`. + +The matrix file +--------------- + +Here is the complete config for this tutorial +(``example-configs/fedsynthetic-demo.yml``): + +.. literalinclude:: ../../example-configs/fedsynthetic-demo.yml + :language: yaml + +Three sections matter: + +- ``strategy.matrix`` — the grid. Every combination of the listed values + is run once: 2 sampling ratios × 2 seeds = 4 experiments. +- ``${{ matrix. }}`` — interpolation. Wherever this pattern appears, + the CLI substitutes the current combination's value. ``seed`` sits at + the top level, so it seeds the dataset *and* the server config of that + run. +- everything else — ``algorithm`` (name plus server/client configs), + ``dataset``, ``model`` — is exactly the config you would pass in + Python, per :doc:`quickstart`, just spelled as YAML. Dataset-specific + constructor arguments (``alpha``, ``beta``, ``iid``, ``num_clients`` + for :class:`~fl_sim.data_processing.fed_synthetic.FedSynthetic`) are + passed straight through. + +Run it +------ + +.. code-block:: bash + + fl-sim example-configs/fedsynthetic-demo.yml + +You will see the four runs execute in sequence, each logging its +combination (``ratio=0.5 seed=0``, ...). Total time: seconds — no +downloads. + +Where the results land +---------------------- + +All runs write into ``~/.cache/fl-sim/.logs/fedsynthetic-demo/`` (the +``log_dir`` of this config). Per run you get one ``.txt`` and one +``.json`` log — the same files :doc:`quickstart` described — plus one +``config.json`` holding the exact server/client configs of the most +recent run (a previous ``config.json`` is rotated to +``config.json.bak``). + +One detail that matters as soon as runs finish within the same minute: +log file names carry a timestamp **plus the run's ``tag``**. Without a +distinct tag per matrix combination, same-named logs overwrite each +other. That is what the ``tag: ratio_${{ ... }}_seed_${{ ... }}`` line +in the config above is for — give every matrix arm its own tag. + +Beyond serial execution +----------------------- + +A matrix of long experiments wants more than one process. The ``strategy`` +section also accepts parallel options (``n_parallel``, or a +``parallel: {mode: parallel_task, num_workers: N}`` block) that dispatch +matrix arms to worker processes — see ``example-configs/multiprocessing-demo.yml`` +for a complete example, and :doc:`parallel_execution` for the in-process +executor backend that parallelizes *within* a single run. The two are +orthogonal: process-level parallelism across matrix arms, +executor-level parallelism across clients. + +Next steps +---------- + +- :doc:`cli` — the full YAML syntax reference and CLI options. +- :doc:`parallel_execution` — speed up each individual run. +- How-to guides on custom algorithms, datasets and optimizers are on + the way; until then, ``test-files/custom_alg.py``, + ``test-files/custom_dataset.py`` and ``test-files/custom_optimizer.py`` + are complete, runnable examples. diff --git a/docs/source/tutorial-first-experiment.rst b/docs/source/tutorial-first-experiment.rst new file mode 100644 index 00000000..98dfc219 --- /dev/null +++ b/docs/source/tutorial-first-experiment.rst @@ -0,0 +1,158 @@ +.. _tutorial-first-experiment: + +Tutorial: your first controlled experiment +========================================== + +:doc:`Quickstart ` ran one training once. A real experiment +compares configurations under identical conditions and reads the results +back. In this tutorial you will compare three algorithms at two sampling +ratios, then dissect what "communication cost" actually means in fl-sim. + +Everything runs on :class:`~fl_sim.data_processing.fed_synthetic.FedSynthetic` +— no downloads, seconds per run. The full script shown below lives at +``test-files/tutorial_first_experiment.py`` and is exercised in CI. + +Prerequisites: :doc:`quickstart` (the three-object model: dataset, +model, server) and :doc:`concepts` (what a round does). + +Set up the comparison +--------------------- + +Six runs: **FedAvg** (plain averaging), **FedProx** (proximal-regularized +local objectives) and **SCAFFOLD** (control variates) — the classic +gradient, proximal and variance-reduction axes — each at **sampling +ratios 0.5 and 1.0**. + +One deliberate choice: the dataset is constructed **once, before any +config or server**. Config constructors re-seed the global RNG as they +are built, so whatever was constructed last would otherwise shift the +sampling stream of everything after it. The order *dataset → config → +server* keeps every run reproducible: + +.. literalinclude:: ../../test-files/tutorial_first_experiment.py + :language: python + :start-after: # --- begin: setup --- + :end-before: # --- end: setup --- + +``final_accuracy`` reads a run's JSON log — the same file format you saw +in :doc:`quickstart`. SCAFFOLD needs its global step size ``lr`` passed +explicitly (its config has no default for it); FedPD will need a +frequency parameter later. Both are per-algorithm server keywords, +collected in ``SERVER_KW``. + +Run and read the results +------------------------ + +The comparison itself is a loop, and the results land in a +:class:`pandas.DataFrame`: + +.. literalinclude:: ../../test-files/tutorial_first_experiment.py + :language: python + :start-after: # --- begin: compare --- + :end-before: # --- end: compare --- + +Output on one machine (your numbers may differ slightly by package +version): + +.. code-block:: text + + algorithm sample_ratio num_communications final_acc + FedAvg 0.5 90 0.8114 + FedAvg 1.0 180 0.8767 + FedProx 0.5 90 0.8776 + FedProx 1.0 180 0.8776 + SCAFFOLD 0.5 90 0.8767 + SCAFFOLD 1.0 180 0.8767 + +Two honest observations about this table: + +- This synthetic task is *easy* — FedProx and SCAFFOLD reach the same + accuracy ceiling as FedAvg at full sampling. Their advantages show up + on harder, more heterogeneous tasks (see the + :doc:`algorithm theory pages ` for when each + helps). +- The accuracy story saturates; the **communication** story does not. + Every run at ratio 0.5 used exactly 90 communications and every run at + ratio 1.0 used exactly 180 — regardless of the algorithm. That number + deserves a closer look, because it is easy to misread. + +What ``num_communications`` counts +---------------------------------- + +``num_communications`` counts **messages, not rounds**: each round adds +one downlink broadcast *per sampled client* plus one uplink reply *per +sampled client*. With 30 clients at ratio 0.5, every round is +15 downlinks + 15 uplinks = 30 messages, so 3 rounds = 90; at ratio 1.0 +that doubles to 180. + +The discriminating check: **3 rounds at full sampling and 6 rounds at +half sampling both land on 180.** The counter scales with +``2 × sampled_clients × rounds``, not with rounds. Reading it as +"communication rounds" would make 90 look like "90 rounds" — off by a +factor of 30. + +The decomposition to keep in mind: + +.. math:: + + \text{FedAvg} = 2 \times \text{sampled} \times \text{rounds} + +(Note also that the ``step`` recorded in the JSON log is a snapshot +taken at the *start* of each round: it trails the final counter by the +last round's full cost — read the final count from the server, as the +scripts here do.) + +Sampling is not communicating +----------------------------- + +All three algorithms above communicate every round. fl-sim also ships +**communication-skipping** algorithms, and they break the +``2 × sampled × rounds`` identity in an instructive way. Run FedPD — +which uploads only every ``int(1/p)``-th round — for five rounds next to +a five-round FedAvg: + +.. literalinclude:: ../../test-files/tutorial_first_experiment.py + :language: python + :start-after: # --- begin: fedpd --- + :end-before: # --- end: fedpd --- + +Output on one machine: + +.. code-block:: text + + FedAvg: 150 communications = 75 downlink + 75 uplink (5 of 5 rounds uploaded) + FedPD: 90 communications = 75 downlink + 15 uplink (1 of 5 rounds uploaded) + +The decomposition says it precisely: **the downlink is identical** (75 = +5 rounds × 15 clients — broadcasting to the sampled set is never +skipped); FedPD saves *uplinks only*. Five FedPD rounds cost the same +90 messages as three FedAvg rounds — the same communication budget +buys two extra local iterations. + +Three practical details when you run this yourself: + +- ``p`` must satisfy **0 < p < 1**. With p = 1 FedPD uploads every round + (degenerating to FedAvg); with p > 1 the upload frequency becomes zero + and the run crashes. The config exposes ``p`` but the derived upload + frequency ``int(1/p)`` always wins — passing a ``comm_freq`` yourself + is silently overwritten. +- **You will see RuntimeWarnings — they are expected, not errors.** The + server logs ``No message received from the clients...`` once per + skipped round ("nobody uploaded this round, so the global model did + not move"), and each skipping client logs ``No parameters received + from server...`` ("I was not given a new global model this round, so I + keep working from my current parameters"). At tutorial scale the + client-side message appears once per (skipped client × skipped round), + so it dominates the console. Both are how FedPD is designed to behave. +- The JSON ``step`` snapshot trails differently per algorithm: for the + five-round FedAvg it ends at 120 (the last round's 30 messages are + unlogged), for FedPD at 75 (its last round never uploads). The final + counts come from ``server._num_communications``. + +Next steps +---------- + +- :doc:`tutorial-cli-matrix` — run the same kind of comparison from the + command line, as a YAML-driven experiment matrix. +- :doc:`parallel_execution` — speed these runs up with a parallel + executor without changing the results. diff --git a/example-configs/fedsynthetic-demo.yml b/example-configs/fedsynthetic-demo.yml new file mode 100644 index 00000000..a2c1e0a8 --- /dev/null +++ b/example-configs/fedsynthetic-demo.yml @@ -0,0 +1,40 @@ +# Demo config for the CLI tutorial: a 4-run experiment matrix on +# synthetic data. Every combination of `clients_sample_ratio` and `seed` +# in `strategy.matrix` is run once. Runs in seconds, no downloads. + +strategy: + matrix: + clients_sample_ratio: + - 0.5 + - 1.0 + seed: + - 0 + - 1 + n_parallel: 1 # number of parallel jobs, not supported yet but kept for future use + +algorithm: + name: FedAvg + server: + num_clients: 30 + clients_sample_ratio: ${{ matrix.clients_sample_ratio }} + num_iters: 3 + log_dir: fedsynthetic-demo + # a distinct tag per matrix combination: log file names carry the + # tag, so runs finishing within the same minute never overwrite + # each other's json/txt logs + tag: ratio_${{ matrix.clients_sample_ratio }}_seed_${{ matrix.seed }} + client: + num_epochs: 1 + batch_size: 32 + +dataset: + name: FedSynthetic # synthetic data: no download, seconds per run + alpha: 1.0 + beta: 1.0 + iid: true + num_clients: 30 + +model: + name: mlp_d1 + +seed: ${{ matrix.seed }} diff --git a/test-files/tutorial_first_experiment.py b/test-files/tutorial_first_experiment.py new file mode 100644 index 00000000..18ecc91a --- /dev/null +++ b/test-files/tutorial_first_experiment.py @@ -0,0 +1,117 @@ +""" +Your first controlled experiment: FedAvg vs FedProx vs SCAFFOLD. + +This script is the source of the code blocks shown in the +``tutorial-first-experiment`` page of the documentation +(``docs/source/tutorial-first-experiment.rst``): the marked sections are +included verbatim via Sphinx ``literalinclude`` directives, so the docs +and this script can never drift apart. The size is pinned to the tutorial +budget (``num_iters <= 5``, ``num_clients <= 30``, one local epoch), +enforced by ``test/test_tutorial_scripts.py``. + +The dataset is built ONCE, before any config/server, on purpose: config +constructors re-seed the global RNG when they run, so whatever was built +last would otherwise shift the results. Construct in the order +dataset -> config -> server and the numbers are stable. +""" + +# --- begin: setup --- +import json +from pathlib import Path + +import pandas as pd + +from fl_sim.algorithms.fedopt import FedAvgClientConfig, FedAvgServer, FedAvgServerConfig +from fl_sim.algorithms.fedpd import FedPDClientConfig, FedPDServer, FedPDServerConfig +from fl_sim.algorithms.fedprox import FedProxClientConfig, FedProxServer, FedProxServerConfig +from fl_sim.algorithms.scaffold import SCAFFOLDClientConfig, SCAFFOLDServer, SCAFFOLDServerConfig +from fl_sim.data_processing.fed_synthetic import FedSynthetic + +ALGORITHMS = { + "FedAvg": (FedAvgServer, FedAvgServerConfig, FedAvgClientConfig), + "FedProx": (FedProxServer, FedProxServerConfig, FedProxClientConfig), + "SCAFFOLD": (SCAFFOLDServer, SCAFFOLDServerConfig, SCAFFOLDClientConfig), + "FedPD": (FedPDServer, FedPDServerConfig, FedPDClientConfig), +} + +# SCAFFOLD's server config requires its global step size, and FedPD's +# upload frequency is 0 < p < 1 (it uploads once every int(1/p) rounds); +# the others use their defaults. +SERVER_KW = {"FedAvg": {}, "FedProx": {}, "SCAFFOLD": {"lr": 0.03}, "FedPD": {"p": 0.3}} + +# one synthetic dataset, split across 30 clients, shared by every run below +dataset = FedSynthetic(1.0, 1.0, True, 30) +model = dataset.candidate_models["mlp_d1"] + + +def run(algorithm, sample_ratio, num_iters, tag, **extra_kw): + """Train one algorithm and return the server. + + Algorithm-specific server keywords (SCAFFOLD's global step size, + FedPD's upload frequency ``p``) come from ``SERVER_KW``; anything + passed as ``extra_kw`` is merged on top. + """ + server_cls, server_config_cls, client_config_cls = ALGORITHMS[algorithm] + server_kw = {**SERVER_KW[algorithm], **extra_kw} + server = server_cls( + model, + dataset, + server_config_cls( + num_iters=num_iters, + num_clients=30, + clients_sample_ratio=sample_ratio, + log_dir=f"t1_{tag}", + **server_kw, + ), + client_config_cls(batch_size=32, num_epochs=1), + ) + server.train() + return server + + +def final_accuracy(server): + """Read the last logged server-side validation accuracy of a run. + + The JSON logger buffers in memory; flush it to materialize the file. + """ + server._logger_manager.flush() + log_dir = Path(server._logger_manager.log_dir) + metrics_file = next(p for p in log_dir.glob("*.json") if p.name != "config.json") + metrics = json.loads(metrics_file.read_text()) + return metrics["val"]["Server"][-1]["acc"] + + +# --- end: setup --- + +# --- begin: compare --- +results = [] +for algorithm in ("FedAvg", "FedProx", "SCAFFOLD"): + for sample_ratio in (0.5, 1.0): + server = run(algorithm, sample_ratio, num_iters=3, tag=f"{algorithm}_{sample_ratio}") + results.append( + { + "algorithm": algorithm, + "sample_ratio": sample_ratio, + "num_communications": server._num_communications, + "final_acc": round(final_accuracy(server), 4), + } + ) + +print(pd.DataFrame(results).to_string(index=False)) +# --- end: compare --- + +# --- begin: fedpd --- +fedavg = run("FedAvg", 0.5, num_iters=5, tag="FedAvg_5it") +fedpd = run("FedPD", 0.5, num_iters=5, tag="FedPD_5it") + +# The skipped rounds emit expected RuntimeWarnings on both the server +# and the client side -- see the tutorial page for what they mean. +for name, server in [("FedAvg", fedavg), ("FedPD", fedpd)]: + sampled = 15 # 30 clients x 0.5 + downlink = sampled * 5 # every round broadcasts to every sampled client + uplink = server._num_communications - downlink + print( + f"{name}: {server._num_communications} communications = " + f"{downlink} downlink + {uplink} uplink ({uplink // sampled} of 5 rounds uploaded)" + ) +# --- end: fedpd --- diff --git a/test/test_tutorial_scripts.py b/test/test_tutorial_scripts.py index b731820d..59e2ac84 100644 --- a/test/test_tutorial_scripts.py +++ b/test/test_tutorial_scripts.py @@ -13,6 +13,8 @@ Each run is isolated: fl-sim's log directory is redirected into a temporary directory, and ``sys.path`` is restored after ``runpy``. +The whole suite exercises real training runs and takes well under a +minute on a laptop. """ import json @@ -30,7 +32,9 @@ TUTORIAL_SCRIPTS = { "quickstart": REPO_ROOT / "test-files" / "quickstart.py", + "tutorial_first_experiment": REPO_ROOT / "test-files" / "tutorial_first_experiment.py", } +CLI_MATRIX_DEMO = REPO_ROOT / "example-configs" / "fedsynthetic-demo.yml" @pytest.fixture() @@ -40,32 +44,37 @@ def isolated_log_dir(tmp_path, monkeypatch): yield tmp_path -def _run_script(script: Path) -> None: +def _run_script(script: Path): """Execute a tutorial script in-process, restoring ``sys.path`` afterwards. ``run_name="__main__"`` is required: without it the scripts' module-level code still runs under ``runpy``, but any ``if __name__ == "__main__":`` guard would silently not execute. + Returns the script's final global namespace for behavioral assertions. """ saved_sys_path = list(sys.path) try: - runpy.run_path(str(script), run_name="__main__") + return runpy.run_path(str(script), run_name="__main__") finally: sys.path[:] = saved_sys_path +def _check_tutorial_budget(log_dir: Path) -> dict: + """Assert the run respected the tutorial size budget, via config.json.""" + config = json.loads((log_dir / "config.json").read_text()) + assert config["server"]["num_iters"] <= 5 + assert config["server"]["num_clients"] <= 30 + assert config["client"]["num_epochs"] == 1 + return config + + def test_quickstart_script_runs_and_respects_tutorial_budget(isolated_log_dir): - script = TUTORIAL_SCRIPTS["quickstart"] - _run_script(script) + _run_script(TUTORIAL_SCRIPTS["quickstart"]) run_dir = isolated_log_dir / "quickstart" assert run_dir.is_dir(), "the quickstart run should log into its own subdirectory" - # tutorial budget, asserted on the values the run actually used - config = json.loads((run_dir / "config.json").read_text()) - assert config["server"]["num_iters"] <= 5 - assert config["server"]["num_clients"] <= 30 - assert config["client"]["num_epochs"] == 1 + _check_tutorial_budget(run_dir) # the docs promise one metrics json + one txt log per run metrics_files = [p for p in run_dir.glob("*.json") if p.name != "config.json"] @@ -78,6 +87,67 @@ def test_quickstart_script_runs_and_respects_tutorial_budget(isolated_log_dir): assert 0.0 <= latest["acc"] <= 1.0 +def test_first_experiment_script_runs_and_produces_the_documented_table(isolated_log_dir): + script = TUTORIAL_SCRIPTS["tutorial_first_experiment"] + script_dir = isolated_log_dir # the script logs into t1_* subdirectories + + namespace = _run_script(script) + + # six comparison runs, each within the tutorial budget + results = namespace["results"] + assert len(results) == 6 + assert {row["algorithm"] for row in results} == {"FedAvg", "FedProx", "SCAFFOLD"} + assert {row["sample_ratio"] for row in results} == {0.5, 1.0} + for row in results: + assert 0.0 <= row["final_acc"] <= 1.0 + # 2 x sampled x rounds: 15 sampled clients x 3 rounds x 2 directions + expected = 90 if row["sample_ratio"] == 0.5 else 180 + assert ( + row["num_communications"] == expected + ), "the docs teach this identity; if it breaks, the tutorial narrative breaks" + for sub in script_dir.iterdir(): + if sub.name.startswith("t1_") and sub.is_dir(): + _check_tutorial_budget(sub) + + # the FedPD coda: fewer communications than FedAvg at the same + # schedule, and at least one real uplink (guards against the degenerate + # p/comm_freq combination where FedPD never uploads and trivially + # "wins" the comparison) + fedavg, fedpd = namespace["fedavg"], namespace["fedpd"] + sampled = 15 + assert fedavg._num_communications == 2 * sampled * 5 == 150 + assert fedpd._num_communications < fedavg._num_communications + downlink = sampled * 5 + assert fedpd._num_communications - downlink > 0, "FedPD should have uploaded at least once" + + +def test_cli_matrix_demo_expands_and_runs(isolated_log_dir): + from fl_sim.cli import parse_config_file, single_run + + configs, _parallel = parse_config_file(CLI_MATRIX_DEMO) + + # 2 ratios x 2 seeds, within the tutorial matrix budget + assert 1 < len(configs) <= 4 + ratios = {cfg["algorithm"]["server"]["clients_sample_ratio"] for cfg in configs} + seeds = {cfg["seed"] for cfg in configs} + assert len(ratios) * len(seeds) == len(configs), "the matrix should expand into distinct combinations" + # interpolation resolved everywhere + for cfg in configs: + assert "${{ matrix." not in str(cfg) + + for cfg in configs: + single_run(cfg) + + run_dir = isolated_log_dir / "fedsynthetic-demo" + # every run must leave its own logs: distinct tags prevent same-minute + # overwrites, which is exactly the pitfall the tutorial teaches + jsons = [p for p in run_dir.glob("*.json") if p.name != "config.json"] + txts = list(run_dir.glob("*.txt")) + assert len(jsons) == len(configs), "each matrix arm should produce its own json log" + assert len(txts) == len(configs) + _check_tutorial_budget(run_dir) + + def test_tutorial_scripts_keep_literalinclude_markers(): """The markers delimit the fragments the docs include; dropping one silently empties a documentation code block."""