From b522c74e41611500bf7fec7e3d1f896e8fa7d824 Mon Sep 17 00:00:00 2001 From: baranwang Date: Mon, 19 Oct 2026 16:01:07 +0800 Subject: [PATCH] ci: mirror a release's engine and models to dl.godwit.email A manual workflow downloads the pinned llama.cpp build for each platform and the models at pinned Hugging Face revisions, checks their hashes, uploads each file once (a path never changes once published) and prints the registry assets. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mirror.yml | 52 +++++++++++++++ scripts/mirror.py | 122 +++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 .github/workflows/mirror.yml create mode 100755 scripts/mirror.py diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml new file mode 100644 index 0000000..4f682a7 --- /dev/null +++ b/.github/workflows/mirror.yml @@ -0,0 +1,52 @@ +# Mirrors a release's engine and models to dl.godwit.email/plugins/startlux/ and +# prints its registry assets. Manual only: without "upload" it downloads, checks +# and prints, writing nothing to the bucket. Needs the R2_* secrets to upload. +name: Mirror assets + +on: + workflow_dispatch: + inputs: + tag: + description: Release tag, e.g. v0.1.0 + required: true + upload: + description: Upload to the bucket (otherwise a dry run) + type: boolean + default: false + +permissions: + contents: read + +jobs: + mirror: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + ref: refs/tags/${{ inputs.tag }} + # The script from this branch: a tag made before it existed can still be mirrored. + - uses: actions/checkout@v4 + with: + path: current + sparse-checkout: scripts/mirror.py + sparse-checkout-cone-mode: false + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - name: Mirror + env: + AWS_ACCESS_KEY_ID: ${{ inputs.upload && secrets.R2_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.upload && secrets.R2_SECRET_ACCESS_KEY || '' }} + AWS_DEFAULT_REGION: auto + R2: https://9c578ed9cad837fb4a2564db7b36b483.r2.cloudflarestorage.com + run: | + cp current/scripts/mirror.py scripts/mirror.py + python scripts/mirror.py "$RUNNER_TEMP/mirror" ${{ inputs.upload && '--upload' || '' }} + - name: Assets + run: | + { echo '```json'; cat "$RUNNER_TEMP/mirror/assets.json"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + with: + name: assets-${{ inputs.tag }} + path: ${{ runner.temp }}/mirror/assets.json diff --git a/scripts/mirror.py b/scripts/mirror.py new file mode 100755 index 0000000..39996ff --- /dev/null +++ b/scripts/mirror.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Mirror the engine and models to dl.godwit.email and print the registry assets. + + mirror.py WORK_DIR [--upload] + +scripts/engine.py lays out the pinned llama.cpp build (third_party/llama/VERSION) +per platform, as Godwit stores the engine asset. The models come from startlux-models on Hugging Face at +pinned revisions and must have the hashes below. Without --upload nothing is +written to the bucket. With it, each file goes to s3://godwit-downloads/plugins/ +startlux/... once: a path that already holds other bytes fails the run, since a +published release's files never change. WORK_DIR/assets.json is the release's +"assets" for registry.json. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.request + +BUCKET = "godwit-downloads" +PUBLIC = "https://dl.godwit.email" +PLATFORMS = ["darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64", "windows-arm64"] +GPU = ["darwin-arm64"] # only Apple silicon has a GPU back end; elsewhere only 2B is offered (CPU) + +HF = "https://huggingface.co/startlux-models/StartLux-Decision-v1.1-" +MODELS = { + "4b-q4": { + "model.gguf": (HF + "4B-Instant-GGUF/resolve/a9ab4199fdff586f86e3baf8c9557f6c5ca0fdc0/StartLux-Decision-v1.1-4B-Instant-Q4_K_M.gguf", + "5fcfe9771db996a868f78215c08955c21cf5e8188609e17e9c653183d946e84d"), + "decision_config.json": (HF + "4B-Instant/resolve/416088f77cfce09633906a3675c7cd2e5dabcf10/decision_config.json", + "9ca014791b5c71996c212c7aed6275405c4d9b682a6b6eba325a3ab17bcccc08"), + }, + "2b-q8": { + "model.gguf": (HF + "2B-Instant-GGUF/resolve/afb95dbba67e648f4e1b5ad3f770a91f799f9496/StartLux-Decision-v1.1-2B-Instant-Q8_0.gguf", + "170cf86482a3340c7bd2000f921aa482820a992c14587f496f43fc57009f9529"), + "decision_config.json": (HF + "2B-Instant/resolve/f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1/decision_config.json", + "3bd3c0c3854b18be967c7a660d38b8340aa2b4c14648d8ce0260cd32f6e89c1a"), + }, +} +# Physical memory each needs, measured (docs/parity.md): Windows commits the +# 4K-token compute buffer that macOS and Linux only reserve. +MEMORY = {"4b-q4": {"default": 8}, "2b-q8": {"default": 6, "windows-*": 12}} + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def download(url, dest, want): + if not (dest.exists() and sha256(dest) == want): + dest.parent.mkdir(parents=True, exist_ok=True) + with urllib.request.urlopen(url, timeout=600) as r, open(dest, "wb") as f: + while chunk := r.read(1 << 20): + f.write(chunk) + if (got := sha256(dest)) != want: + sys.exit(f"{url}: sha256 {got}, not {want}") + + +def aws(*args, check=True): + return subprocess.run(["aws", *args, "--endpoint-url", os.environ["R2"]], check=check, capture_output=True, text=True) + + +def publish(local, key, digest, upload): + """The registry file for local at key: uploaded once, never replaced.""" + if upload: + head = aws("s3api", "head-object", "--bucket", BUCKET, "--key", key, check=False) + if head.returncode == 0: + have = json.loads(head.stdout).get("Metadata", {}).get("sha256") + if have != digest: + sys.exit(f"{key} already holds other bytes ({have}); a published file never changes") + print(f"{key}: already there") + else: + aws("s3", "cp", str(local), f"s3://{BUCKET}/{key}", "--metadata", f"sha256={digest}", + "--content-type", "application/octet-stream", "--cache-control", "public, max-age=31536000, immutable") + print(f"{key}: uploaded") + return {"name": local.name, "url": f"{PUBLIC}/{key}", "sha256": digest, "size": local.stat().st_size} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("work", type=Path) + ap.add_argument("--upload", action="store_true") + args = ap.parse_args() + if args.upload and not os.environ.get("R2"): + sys.exit("--upload needs R2 (the bucket's endpoint) and the AWS_* credentials") + root = Path(__file__).resolve().parent.parent + version = (root / "third_party" / "llama" / "VERSION").read_text().strip() + + assets = [] + for platform in PLATFORMS: + out = args.work / "engine" / platform + if not out.exists(): + subprocess.run([sys.executable, str(root / "scripts" / "engine.py"), str(out), "--platform", platform], check=True) + files = [publish(p, f"plugins/startlux/engine/{version}/{platform}/{p.name}", sha256(p), args.upload) + for p in sorted(out.iterdir())] + assets.append({"name": "engine", "platforms": [platform], "variants": [{"id": version, "files": files}]}) + + variants = {} + for variant, files in MODELS.items(): + published = [] + for name, (url, want) in files.items(): + local = args.work / "models" / variant / name + download(url, local, want) + published.append(publish(local, f"plugins/startlux/models/v1.1/{variant}/{name}", want, args.upload)) + variants[variant] = {"id": variant, "minMemoryGB": MEMORY[variant], "files": published} + cpu = [p for p in PLATFORMS if p not in GPU] + assets.append({"name": "model", "platforms": GPU, "variants": [variants["4b-q4"], variants["2b-q8"]]}) + assets.append({"name": "model", "platforms": cpu, "variants": [variants["2b-q8"]]}) + + (args.work / "assets.json").write_text(json.dumps(assets, indent=2) + "\n") + print(f"assets: {args.work / 'assets.json'}") + + +if __name__ == "__main__": + main()