Repository navigation
parse synom@0.11.3 (c36970d3-a0f8-40a2-a7b8-53287a4ad2f9) #4901
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: parse | |
| run-name: >- | |
| ${{ inputs.crate != '' | |
| && format('parse {0}@{1} ({2})', inputs.crate, inputs.version, inputs.request_id) | |
| || format('parse scheduled backfill ({0})', github.run_id) }} | |
| # Bounded static R2 artifact publisher. | |
| # | |
| # The site build only builds the Cloudflare Worker/Svelte app. Parsing is | |
| # handled offline on GitHub-hosted runners: | |
| # | |
| # build -> compile codeview-cli once; every other job downloads it | |
| # parse-one -> exact crate/version request from the hosted queue | |
| # plan -> read requested work and write one deterministic work plan to R2 | |
| # parse-shard -> drain bounded shards with docs.rs request spacing | |
| # freshness-merge -> single-writer index/catalog/ref finalizer | |
| # | |
| # Hosted requests run concurrently with each other and with the nightly | |
| # backfill; only freshness-merge (the catalog writer) is serialized. | |
| # | |
| # Local/offline equivalent: | |
| # | |
| # STATIC_R2_TARGET=remote CODEVIEW_SKIP_SIDECAR=1 \ | |
| # cargo run --release -p codeview-cli -- cron publish-run \ | |
| # --corpus top:100 --max-total 100 --shard-count 10 --concurrency 4 | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| crate: | |
| description: 'Exact crate name for a queued hosted parse request' | |
| required: false | |
| default: '' | |
| version: | |
| description: 'Exact crate version or std toolchain/channel for a queued hosted parse request' | |
| required: false | |
| default: '' | |
| request_kind: | |
| description: '`crate` for docs.rs JSON, `sysroot` for std/core/alloc/proc_macro/test' | |
| required: false | |
| default: 'crate' | |
| toolchain: | |
| description: 'Rust toolchain for sysroot requests; defaults to version' | |
| required: false | |
| default: '' | |
| parse_force: | |
| description: 'Force exact crate/version reparse' | |
| required: false | |
| default: 'false' | |
| workflow_id: | |
| description: 'Cloudflare Workflow instance id to notify' | |
| required: false | |
| default: '' | |
| request_id: | |
| description: 'Hosted parse request id' | |
| required: false | |
| default: '' | |
| callback_url: | |
| description: 'Cloudflare parser callback URL' | |
| required: false | |
| default: '' | |
| corpus: | |
| description: "`catalog`, `top:N`, `file:<path>`, or `all`" | |
| required: false | |
| default: 'top:5000' | |
| max_total: | |
| description: 'Maximum crates in the plan; shards drain max_per_shard each, the hosted queue dispatches the rest' | |
| required: false | |
| default: '1000' | |
| shard_count: | |
| description: 'Deterministic publish shards' | |
| required: false | |
| default: '8' | |
| max_per_shard: | |
| description: 'Maximum crates each shard can drain' | |
| required: false | |
| default: '15' | |
| max_duration_minutes: | |
| description: 'Wall-clock cap for each shard' | |
| required: false | |
| default: '45' | |
| docsrs_min_delay_ms: | |
| description: 'Minimum delay between docs.rs requests per shard' | |
| required: false | |
| default: '15000' | |
| force: | |
| description: 'Comma-separated crate names to reparse' | |
| required: false | |
| default: '' | |
| bootstrap: | |
| description: 'Initialize the aggregate index from per-version freshness records' | |
| required: false | |
| default: 'false' | |
| env: | |
| R2_BUCKET: crate-graphs | |
| STATIC_R2_TARGET: remote | |
| CODEVIEW_SKIP_SIDECAR: '1' | |
| permissions: | |
| contents: read | |
| # Exact-crate requests get their own group so a user request never waits | |
| # behind the ~45 minute backfill. Backfills still never overlap. | |
| concurrency: | |
| group: >- | |
| ${{ github.event_name == 'workflow_dispatch' && inputs.crate != '' | |
| && format('codeview-parse-{0}-{1}-{2}', inputs.request_kind || 'crate', inputs.crate, inputs.version) | |
| || 'codeview-static-publish' }} | |
| cancel-in-progress: false | |
| queue: max | |
| jobs: | |
| build: | |
| name: build codeview-cli | |
| runs-on: ubuntu-latest | |
| outputs: | |
| parser_revision: ${{ steps.revision.outputs.parser_revision }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Install Rust stable | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cron-release | |
| - name: Build codeview-cli | |
| run: cargo +stable build --locked --release -p codeview-cli | |
| - name: Resolve parser revision | |
| id: revision | |
| run: | | |
| bash .github/scripts/export-parser-revision.sh | |
| grep '^PARSER_REVISION=' "$GITHUB_ENV" | tail -1 | sed 's/^PARSER_REVISION=/parser_revision=/' >> "$GITHUB_OUTPUT" | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: codeview-cli | |
| path: target/release/codeview | |
| retention-days: 1 | |
| if-no-files-found: error | |
| parse_one: | |
| name: parse exact crate | |
| needs: build | |
| if: ${{ github.event_name == 'workflow_dispatch' && (inputs.request_kind == '' || inputs.request_kind == 'crate') && inputs.crate != '' && inputs.version != '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| exit_code: ${{ steps.parse.outputs.exit_code }} | |
| run_id: ${{ steps.parse.outputs.run_id }} | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| steps: | |
| # No repository checkout: the docs.rs fallback builds untrusted crate | |
| # source, so keep tokens and the repo off this runner's disk. | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: nightly | |
| - name: Parse and publish exact crate | |
| id: parse | |
| env: | |
| NAME: ${{ inputs.crate }} | |
| VERSION: ${{ inputs.version }} | |
| PARSE_FORCE: ${{ inputs.parse_force || 'false' }} | |
| run: | | |
| chmod +x bin/codeview | |
| RUN_ID="direct-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" | |
| FORCE_FLAG="" | |
| if [[ "$PARSE_FORCE" == "true" ]]; then | |
| FORCE_FLAG="--force" | |
| fi | |
| for ATTEMPT in 1 2 3; do | |
| set +e | |
| bin/codeview cron parse-one \ | |
| --name "$NAME" \ | |
| --version "$VERSION" \ | |
| $FORCE_FLAG \ | |
| --run-id "$RUN_ID" \ | |
| --bucket "$R2_BUCKET" | |
| EXIT_CODE=$? | |
| set -e | |
| if [[ "$EXIT_CODE" -eq 0 || "$EXIT_CODE" -ne 64 || "$ATTEMPT" -eq 3 ]]; then | |
| break | |
| fi | |
| DELAY=$((15 * 2 ** (ATTEMPT - 1))) | |
| echo "Transient parse failure (attempt $ATTEMPT); retrying in ${DELAY}s" | |
| sleep "$DELAY" | |
| done | |
| echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" | |
| exit "$EXIT_CODE" | |
| merge_one: | |
| name: refresh hosted refs and catalog | |
| needs: [build, parse_one] | |
| # Also runs after a failed parse so the failure is recorded and the | |
| # planner stops retrying deterministic failures. | |
| if: ${{ always() && needs.build.result == 'success' && (needs.parse_one.result == 'success' || needs.parse_one.result == 'failure') && needs.parse_one.outputs.run_id != '' }} | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: codeview-catalog-writer | |
| cancel-in-progress: false | |
| queue: max | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Merge direct delta | |
| run: | | |
| chmod +x bin/codeview | |
| bin/codeview cron freshness-merge \ | |
| --run-id "${{ needs.parse_one.outputs.run_id }}" \ | |
| --write-catalog \ | |
| --bootstrap-if-missing \ | |
| --bucket "$R2_BUCKET" | |
| seed_sysroot: | |
| name: seed std sysroot | |
| needs: build | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.request_kind == 'sysroot' && inputs.crate != '' && inputs.version != '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_id: ${{ steps.seed.outputs.run_id }} | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| steps: | |
| - name: Generate exact rustdoc JSON | |
| id: rustdoc_json | |
| env: | |
| REQUEST_VERSION: ${{ inputs.version }} | |
| REQUEST_TOOLCHAIN: ${{ inputs.toolchain }} | |
| run: | | |
| TOOLCHAIN="${REQUEST_TOOLCHAIN:-$REQUEST_VERSION}" | |
| if [[ ! "$TOOLCHAIN" =~ ^(stable|beta|nightly)$ ]]; then | |
| echo "::error::sysroot toolchain must be stable, beta, or nightly" | |
| exit 1 | |
| fi | |
| rustup toolchain install "$TOOLCHAIN" --profile minimal | |
| if [[ "$TOOLCHAIN" == "nightly" ]]; then | |
| rustup component add rust-docs-json --toolchain "$TOOLCHAIN" | |
| SYSROOT="$(rustup run "$TOOLCHAIN" rustc --print sysroot)" | |
| JSON_DIR="$SYSROOT/share/doc/rust/json" | |
| else | |
| RUST_COMMIT="$(rustup run "$TOOLCHAIN" rustc -vV | awk '/^commit-hash:/ { print $2 }')" | |
| if [[ ! "$RUST_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then | |
| echo "::error::could not resolve the exact $TOOLCHAIN rustc commit" | |
| exit 1 | |
| fi | |
| git init rust-source | |
| git -C rust-source remote add origin https://github.com/rust-lang/rust.git | |
| git -C rust-source -c protocol.version=2 fetch \ | |
| --depth=1 --filter=blob:none origin "$RUST_COMMIT" | |
| git -C rust-source checkout --detach FETCH_HEAD | |
| ( | |
| cd rust-source | |
| ./x doc --stage 0 --json library | |
| ) | |
| JSON_DIR="$(find rust-source/build -type d -name json-doc -print -quit)" | |
| fi | |
| for CRATE in std core alloc proc_macro test; do | |
| test -s "$JSON_DIR/$CRATE.json" || { | |
| echo "::error::missing generated $CRATE.json in $JSON_DIR" | |
| exit 1 | |
| } | |
| done | |
| echo "json_dir=$JSON_DIR" >> "$GITHUB_OUTPUT" | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Seed and publish std artifacts | |
| id: seed | |
| env: | |
| REQUEST_VERSION: ${{ inputs.version }} | |
| REQUEST_TOOLCHAIN: ${{ inputs.toolchain }} | |
| JSON_DIR: ${{ steps.rustdoc_json.outputs.json_dir }} | |
| PARSE_FORCE: ${{ inputs.parse_force || 'false' }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| run: | | |
| chmod +x bin/codeview | |
| RUN_ID="sysroot-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" | |
| TOOLCHAIN="${REQUEST_TOOLCHAIN:-$REQUEST_VERSION}" | |
| FORCE_FLAG="" | |
| if [[ "$PARSE_FORCE" == "true" ]]; then | |
| FORCE_FLAG="--force" | |
| fi | |
| bin/codeview cron seed-std \ | |
| --toolchains "$TOOLCHAIN" \ | |
| --json-dir "$JSON_DIR" \ | |
| $FORCE_FLAG \ | |
| --run-id "$RUN_ID" \ | |
| --bucket "$R2_BUCKET" | |
| merge_sysroot: | |
| name: refresh hosted sysroot refs and catalog | |
| needs: [build, seed_sysroot] | |
| if: ${{ always() && needs.build.result == 'success' && (needs.seed_sysroot.result == 'success' || needs.seed_sysroot.result == 'failure') && needs.seed_sysroot.outputs.run_id != '' }} | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: codeview-catalog-writer | |
| cancel-in-progress: false | |
| queue: max | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Merge sysroot delta | |
| run: | | |
| chmod +x bin/codeview | |
| bin/codeview cron freshness-merge \ | |
| --run-id "${{ needs.seed_sysroot.outputs.run_id }}" \ | |
| --write-catalog \ | |
| --bootstrap-if-missing \ | |
| --bucket "$R2_BUCKET" | |
| plan: | |
| name: plan static publish | |
| needs: build | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.crate == '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| worker_matrix: ${{ steps.plan.outputs.worker_matrix }} | |
| run_id: ${{ steps.plan.outputs.run_id }} | |
| shard_count: ${{ steps.plan.outputs.shard_count }} | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Build bounded work plan | |
| id: plan | |
| env: | |
| CORPUS: ${{ inputs.corpus || 'top:5000' }} | |
| MAX_TOTAL: ${{ inputs.max_total || '1000' }} | |
| MAX_PER_SHARD: ${{ inputs.max_per_shard || '15' }} | |
| SHARD_COUNT: ${{ inputs.shard_count || '8' }} | |
| FORCE_NAMES: ${{ inputs.force || '' }} | |
| run: | | |
| chmod +x bin/codeview | |
| RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| PLAN_KEY="rust/_runs/${RUN_ID}/plan.json" | |
| # Shards drain MAX_PER_SHARD items each. The plan holds up to | |
| # MAX_TOTAL so the hosted queue can dispatch the remainder. | |
| PLAN_PER_SHARD=$(( (MAX_TOTAL + SHARD_COUNT - 1) / SHARD_COUNT )) | |
| if [ "$PLAN_PER_SHARD" -lt "$MAX_PER_SHARD" ]; then PLAN_PER_SHARD="$MAX_PER_SHARD"; fi | |
| bin/codeview cron plan \ | |
| --mode daily \ | |
| --corpus "$CORPUS" \ | |
| --max-total "$MAX_TOTAL" \ | |
| --max-per-shard "$PLAN_PER_SHARD" \ | |
| --shard-drain-limit "$MAX_PER_SHARD" \ | |
| --shard-count "$SHARD_COUNT" \ | |
| --force "$FORCE_NAMES" \ | |
| --plan-out "r2:${PLAN_KEY}" \ | |
| --matrix-out "$GITHUB_OUTPUT" \ | |
| --run-id "$RUN_ID" \ | |
| --bucket "$R2_BUCKET" | |
| parse: | |
| name: parse shard ${{ matrix.shard_index }} | |
| needs: [build, plan] | |
| if: needs.plan.outputs.worker_matrix != '' && needs.plan.outputs.worker_matrix != '[]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: ${{ fromJSON(needs.plan.outputs.worker_matrix) }} | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: nightly | |
| - name: Drain shard | |
| env: | |
| MAX_PER_SHARD: ${{ inputs.max_per_shard || '15' }} | |
| MAX_DURATION_MINUTES: ${{ inputs.max_duration_minutes || '45' }} | |
| DOCSRS_MIN_DELAY_MS: ${{ inputs.docsrs_min_delay_ms || '15000' }} | |
| run: | | |
| chmod +x bin/codeview | |
| PLAN_KEY="rust/_runs/${{ needs.plan.outputs.run_id }}/plan.json" | |
| bin/codeview cron parse-shard \ | |
| --plan "r2:${PLAN_KEY}" \ | |
| --shard-index "${{ matrix.shard_index }}" \ | |
| --shard-count "${{ needs.plan.outputs.shard_count }}" \ | |
| --max-items "$MAX_PER_SHARD" \ | |
| --max-duration-minutes "$MAX_DURATION_MINUTES" \ | |
| --docsrs-min-delay-ms "$DOCSRS_MIN_DELAY_MS" \ | |
| --run-id "${{ needs.plan.outputs.run_id }}" \ | |
| --bucket "$R2_BUCKET" | |
| finalize: | |
| name: merge freshness + catalog | |
| needs: [build, plan, parse] | |
| if: always() && needs.build.result == 'success' && needs.plan.outputs.worker_matrix != '' && needs.plan.outputs.worker_matrix != '[]' | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: codeview-catalog-writer | |
| cancel-in-progress: false | |
| queue: max | |
| env: | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: codeview-cli | |
| path: bin | |
| - name: Merge deltas and publish catalog | |
| env: | |
| BOOTSTRAP: ${{ inputs.bootstrap || 'false' }} | |
| run: | | |
| chmod +x bin/codeview | |
| BOOTSTRAP_FLAG="" | |
| if [[ "$BOOTSTRAP" == "true" ]]; then | |
| BOOTSTRAP_FLAG="--bootstrap" | |
| fi | |
| bin/codeview cron freshness-merge \ | |
| --run-id "${{ needs.plan.outputs.run_id }}" \ | |
| --write-catalog \ | |
| $BOOTSTRAP_FLAG \ | |
| --bucket "$R2_BUCKET" | |
| notify_crate: | |
| name: notify hosted crate workflow | |
| needs: [build, parse_one, merge_one] | |
| if: ${{ always() && github.event_name == 'workflow_dispatch' && (inputs.request_kind == '' || inputs.request_kind == 'crate') && inputs.callback_url != '' && inputs.workflow_id != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Send callback | |
| env: | |
| CALLBACK_URL: ${{ inputs.callback_url }} | |
| CALLBACK_SECRET: ${{ secrets.PARSE_CALLBACK_SECRET }} | |
| WORKFLOW_ID: ${{ inputs.workflow_id }} | |
| REQUEST_ID: ${{ inputs.request_id }} | |
| REQUEST_KIND: crate | |
| CRATE_NAME: ${{ inputs.crate }} | |
| CRATE_VERSION: ${{ inputs.version }} | |
| JOB_RESULT: ${{ needs.parse_one.result }} | |
| MERGE_RESULT: ${{ needs.merge_one.result }} | |
| PARSE_EXIT_CODE: ${{ needs.parse_one.outputs.exit_code }} | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [[ -z "$CALLBACK_SECRET" ]]; then | |
| echo "::error::PARSE_CALLBACK_SECRET is not configured" | |
| exit 1 | |
| fi | |
| OK=false | |
| ERROR="" | |
| # Exit code 65 means deterministic failure: the hosted queue | |
| # should not retry until the parser revision changes. | |
| PERMANENT=false | |
| if [[ "$PARSE_EXIT_CODE" == "65" ]]; then | |
| PERMANENT=true | |
| fi | |
| if [[ "$JOB_RESULT" == "success" && "$MERGE_RESULT" == "success" ]]; then | |
| OK=true | |
| elif [[ "$JOB_RESULT" == "success" ]]; then | |
| ERROR="GitHub catalog merge job result: $MERGE_RESULT" | |
| else | |
| ERROR="GitHub parse job result: $JOB_RESULT" | |
| fi | |
| jq -n \ | |
| --arg kind "$REQUEST_KIND" \ | |
| --arg workflowId "$WORKFLOW_ID" \ | |
| --arg requestId "$REQUEST_ID" \ | |
| --arg name "$CRATE_NAME" \ | |
| --arg version "$CRATE_VERSION" \ | |
| --arg runId "$RUN_ID" \ | |
| --arg runUrl "$RUN_URL" \ | |
| --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg error "$ERROR" \ | |
| --arg parserRevision "$PARSER_REVISION" \ | |
| --argjson ok "$OK" \ | |
| --argjson permanent "$PERMANENT" \ | |
| '{ | |
| schemaVersion: 1, | |
| kind: $kind, | |
| workflowId: $workflowId, | |
| requestId: $requestId, | |
| name: $name, | |
| version: $version, | |
| ok: $ok, | |
| runId: $runId, | |
| runUrl: $runUrl, | |
| error: (if $error == "" then null else $error end), | |
| permanent: $permanent, | |
| parserRevision: (if $parserRevision == "" then null else $parserRevision end), | |
| completedAt: $completedAt | |
| }' > callback.json | |
| curl -fsS \ | |
| --retry 5 \ | |
| --retry-all-errors \ | |
| --retry-delay 5 \ | |
| --connect-timeout 10 \ | |
| --max-time 60 \ | |
| -X POST "$CALLBACK_URL" \ | |
| -H "content-type: application/json" \ | |
| -H "authorization: Bearer $CALLBACK_SECRET" \ | |
| --data-binary @callback.json | |
| notify_sysroot: | |
| name: notify hosted sysroot workflow | |
| needs: [build, seed_sysroot, merge_sysroot] | |
| if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.request_kind == 'sysroot' && inputs.callback_url != '' && inputs.workflow_id != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Send callback | |
| env: | |
| CALLBACK_URL: ${{ inputs.callback_url }} | |
| CALLBACK_SECRET: ${{ secrets.PARSE_CALLBACK_SECRET }} | |
| WORKFLOW_ID: ${{ inputs.workflow_id }} | |
| REQUEST_ID: ${{ inputs.request_id }} | |
| REQUEST_KIND: sysroot | |
| CRATE_NAME: ${{ inputs.crate }} | |
| CRATE_VERSION: ${{ inputs.version }} | |
| JOB_RESULT: ${{ needs.seed_sysroot.result }} | |
| MERGE_RESULT: ${{ needs.merge_sysroot.result }} | |
| PARSE_EXIT_CODE: '' | |
| PARSER_REVISION: ${{ needs.build.outputs.parser_revision }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [[ -z "$CALLBACK_SECRET" ]]; then | |
| echo "::error::PARSE_CALLBACK_SECRET is not configured" | |
| exit 1 | |
| fi | |
| OK=false | |
| ERROR="" | |
| # Exit code 65 means deterministic failure: the hosted queue | |
| # should not retry until the parser revision changes. | |
| PERMANENT=false | |
| if [[ "$PARSE_EXIT_CODE" == "65" ]]; then | |
| PERMANENT=true | |
| fi | |
| if [[ "$JOB_RESULT" == "success" && "$MERGE_RESULT" == "success" ]]; then | |
| OK=true | |
| elif [[ "$JOB_RESULT" == "success" ]]; then | |
| ERROR="GitHub catalog merge job result: $MERGE_RESULT" | |
| else | |
| ERROR="GitHub sysroot seed job result: $JOB_RESULT" | |
| fi | |
| jq -n \ | |
| --arg kind "$REQUEST_KIND" \ | |
| --arg workflowId "$WORKFLOW_ID" \ | |
| --arg requestId "$REQUEST_ID" \ | |
| --arg name "$CRATE_NAME" \ | |
| --arg version "$CRATE_VERSION" \ | |
| --arg runId "$RUN_ID" \ | |
| --arg runUrl "$RUN_URL" \ | |
| --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg error "$ERROR" \ | |
| --arg parserRevision "$PARSER_REVISION" \ | |
| --argjson ok "$OK" \ | |
| --argjson permanent "$PERMANENT" \ | |
| '{ | |
| schemaVersion: 1, | |
| kind: $kind, | |
| workflowId: $workflowId, | |
| requestId: $requestId, | |
| name: $name, | |
| version: $version, | |
| ok: $ok, | |
| runId: $runId, | |
| runUrl: $runUrl, | |
| error: (if $error == "" then null else $error end), | |
| permanent: $permanent, | |
| parserRevision: (if $parserRevision == "" then null else $parserRevision end), | |
| completedAt: $completedAt | |
| }' > callback.json | |
| curl -fsS \ | |
| --retry 5 \ | |
| --retry-all-errors \ | |
| --retry-delay 5 \ | |
| --connect-timeout 10 \ | |
| --max-time 60 \ | |
| -X POST "$CALLBACK_URL" \ | |
| -H "content-type: application/json" \ | |
| -H "authorization: Bearer $CALLBACK_SECRET" \ | |
| --data-binary @callback.json | |
| summary: | |
| name: parse summary | |
| needs: [build, plan, parse, finalize, parse_one, merge_one, seed_sysroot, merge_sysroot, notify_crate, notify_sysroot] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Echo summary | |
| run: | | |
| echo "run_id=${{ needs.plan.outputs.run_id }}" | |
| echo "shard_count=${{ needs.plan.outputs.shard_count }}" | |
| echo "matrix=${{ needs.plan.outputs.worker_matrix }}" | |
| echo "build_result=${{ needs.build.result }}" | |
| echo "parser_revision=${{ needs.build.outputs.parser_revision }}" | |
| echo "parse_one_result=${{ needs.parse_one.result }}" | |
| echo "merge_one_result=${{ needs.merge_one.result }}" | |
| echo "seed_sysroot_result=${{ needs.seed_sysroot.result }}" | |
| echo "merge_sysroot_result=${{ needs.merge_sysroot.result }}" | |
| echo "notify_crate_result=${{ needs.notify_crate.result }}" | |
| echo "notify_sysroot_result=${{ needs.notify_sysroot.result }}" | |
| echo "parse_result=${{ needs.parse.result }}" | |
| echo "finalize_result=${{ needs.finalize.result }}" |