diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 770d7e3..dbc60d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,11 @@ -# Checks this repo: lints every workflow and template, and runs the shared -# workflows against the small projects in tests/. The deploy, release and -# Terraform workflows need AWS, so only actionlint checks them. +# Checks this repo: lints every workflow and template, and builds a small +# image with the shared image workflow. The deploy and release workflows need +# AWS, so only actionlint checks them. name: CI on: pull_request: + types: [opened, synchronize, reopened, ready_for_review] push: branches: [main] @@ -18,61 +19,15 @@ permissions: jobs: actionlint: name: actionlint + if: github.event.pull_request.draft != true runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: Checking out the code uses: actions/checkout@v7 - - name: Linting the workflows and templates - run: | - bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.12/scripts/download-actionlint.bash) 1.7.12 - ./actionlint -color .github/workflows/*.yml - - # Lints the templates as a repo holds them: in .github/workflows, - # with the Deploy template saved as deploy.yml, and calling the - # shared workflows of this commit, so a wrong input name fails here. - dir="$RUNNER_TEMP/templates" - mkdir -p "$dir/.github/workflows" - cp .github/workflows/{go,python,image,deploy-ecs,release,terraform}.yml "$dir/.github/workflows/" - for f in workflow-templates/*.yml; do - sed 's#probe-lab/.github/.github/workflows/\(.*\)\.yml@v1#./.github/workflows/\1.yml#' "$f" \ - > "$dir/.github/workflows/template-$(basename "$f")" - done - cp "$dir/.github/workflows/template-service-deploy.yml" "$dir/.github/workflows/deploy.yml" - git -C "$dir" init -q - cd "$dir" && "$GITHUB_WORKSPACE/actionlint" -color .github/workflows/template-*.yml .github/workflows/deploy.yml - - go: - uses: ./.github/workflows/go.yml - with: - runner: ubuntu-latest - working-directory: tests/go - clickhouse: true - golangci-lint: v2.15 - lint-command: just lint && golangci-lint run ./... - - python: - uses: ./.github/workflows/python.yml - with: - runner: ubuntu-latest - working-directory: tests/python - clickhouse: true + - name: Setting up just + uses: extractions/setup-just@v4 - image: - uses: ./.github/workflows/image.yml - with: - runner: ubuntu-latest - service: fixture - context: tests/go - dockerfile: tests/go/Dockerfile - - ci-ok: - name: ci-ok - if: always() - needs: [actionlint, go, python, image] - runs-on: ubuntu-latest - steps: - - name: Checking that every job passed - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 + - name: Linting the workflows and templates + run: just lint diff --git a/.github/workflows/deploy-ecs.yml b/.github/workflows/deploy-ecs.yml index 41fb3d5..1c0882a 100644 --- a/.github/workflows/deploy-ecs.yml +++ b/.github/workflows/deploy-ecs.yml @@ -1,11 +1,13 @@ -# Deploys the image of one commit to one ECS service. A person starts it with -# a repo's Deploy workflow. A merge deploys only when its pull request has -# the label that pr-label names. See the README for an example call. +# Deploys the image of one commit to one ECS service. Two events start it in +# the caller: a person pressing the Deploy button (workflow_dispatch), and the +# CI run of a merge (push), where the deploy is the last job, after the image +# job. The button deploys without a check. A merge deploys only when its +# pull request has the label deploy. See the README for an example call. # # It deploys only a commit on the default branch whose image -# -sha is in ECR. It never builds an image: while CI still -# builds it, the deploy waits up to image-wait-minutes. The caller grants -# contents: read and id-token: write, and pull-requests: read for pr-label. +# -sha is in ECR. It never builds an image and never waits +# for one. The caller grants contents: read, id-token: write and +# pull-requests: read. name: Deploy to ECS on: @@ -19,12 +21,6 @@ on: description: The commit to deploy. Empty means the newest on the default branch. type: string default: "" - pr-label: - description: >- - Deploys only if the merged pull request of the commit has this - label, for example deploy. Empty means deploy without a check. - type: string - default: "" cluster: description: The ECS cluster, for example prod-use1-web-ecs-cluster. type: string @@ -53,35 +49,6 @@ on: tasks that something else starts. type: boolean default: false - environment: - type: string - default: production - wait-minutes: - description: How long to wait for the service to be stable. - type: number - default: 15 - image-wait-minutes: - description: >- - How long to wait for the image of the commit while CI still builds - it. 0 means no wait. - type: number - default: 10 - smoke-url: - description: A URL that must answer with a 2xx status after the deploy. Empty means no smoke check. - type: string - default: "" - aws-region: - type: string - default: us-east-1 - aws-role: - type: string - default: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role - ecr-registry: - type: string - default: 019120760881.dkr.ecr.us-east-1.amazonaws.com - ecr-repository: - type: string - default: probelab outputs: commit: description: The full commit that was deployed. @@ -93,12 +60,19 @@ on: description: The new task definition. value: ${{ jobs.deploy.outputs.task-definition-arn }} +env: + AWS_REGION: us-east-1 + AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role + ECR_REGISTRY: 019120760881.dkr.ecr.us-east-1.amazonaws.com + ECR_REPOSITORY: probelab + SERVICE_WAIT_MINUTES: 15 + jobs: # A separate job, so that a merge without the label creates no deployment - # in GitHub. + # in GitHub. The button skips it. label: name: Check the pull request label - if: inputs.pr-label != '' + if: github.event_name == 'push' runs-on: ubuntu-latest timeout-minutes: 5 outputs: @@ -109,8 +83,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} - SHA: ${{ inputs.commit || github.sha }} - LABEL: ${{ inputs.pr-label }} + SHA: ${{ github.sha }} + LABEL: deploy run: | set -euo pipefail prs=$(gh api "repos/$REPO/commits/$SHA/pulls" | jq -r --arg label "$LABEL" \ @@ -131,7 +105,7 @@ jobs: # Deploys hold prod credentials, so they never run on a self-hosted runner. runs-on: ubuntu-latest timeout-minutes: 45 - environment: ${{ inputs.environment }} + environment: production # One rollout at a time for each service. A newer deploy waits, and an # older deploy that still waits is dropped. concurrency: @@ -148,7 +122,7 @@ jobs: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch || 'main' }} - COMMIT: ${{ inputs.commit }} + COMMIT: ${{ inputs.commit || (github.event_name == 'push' && github.sha) || '' }} run: | set -euo pipefail sha=$(gh api "repos/$REPO/commits/${COMMIT:-$DEFAULT_BRANCH}" --jq .sha) @@ -164,34 +138,29 @@ jobs: - name: Configuring AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: - aws-region: ${{ inputs.aws-region }} - role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ env.AWS_ROLE }} - # CI pushes the image after the tests pass. A deploy started right - # after a merge waits for it. + # The image job of the same CI run pushed it before this job started. A + # person who presses the button on a commit without an image gets the + # error at once. - name: Finding the image id: image env: - REPOSITORY: ${{ inputs.ecr-repository }} - IMAGE: ${{ inputs.ecr-registry }}/${{ inputs.ecr-repository }} + REPOSITORY: ${{ env.ECR_REPOSITORY }} + IMAGE: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }} SERVICE: ${{ inputs.service }} SHA: ${{ steps.commit.outputs.sha }} - WAIT_MINUTES: ${{ inputs.image-wait-minutes }} run: | tag="$SERVICE-sha${SHA::7}" - deadline=$((SECONDS + WAIT_MINUTES * 60)) - until aws ecr describe-images --repository-name "$REPOSITORY" --image-ids imageTag="$tag" > /dev/null 2> ecr-error.txt; do - if ! grep -q ImageNotFoundException ecr-error.txt; then + if ! aws ecr describe-images --repository-name "$REPOSITORY" --image-ids imageTag="$tag" > /dev/null 2> ecr-error.txt; then + if grep -q ImageNotFoundException ecr-error.txt; then + echo "::error::There is no image $tag. Check that the CI run of this commit on the default branch passed." + else echo "::error::$(cat ecr-error.txt)" - exit 1 - fi - if (( SECONDS >= deadline )); then - echo "::error::There is no image $tag after $WAIT_MINUTES minutes. Check that the CI run of this commit on the default branch passed." - exit 1 fi - echo "$tag is not in ECR yet. CI may still be running. Checking again in 20 s." - sleep 20 - done + exit 1 + fi echo "image=$IMAGE:$tag" >> "$GITHUB_OUTPUT" # Takes the newest task definition, so a change that Terraform made to @@ -230,7 +199,7 @@ jobs: cluster: ${{ inputs.cluster }} service: ${{ inputs.ecs-service }} wait-for-service-stability: true - wait-for-minutes: ${{ inputs.wait-minutes }} + wait-for-minutes: ${{ env.SERVICE_WAIT_MINUTES }} propagate-tags: SERVICE enable-ecs-managed-tags: true @@ -241,19 +210,3 @@ jobs: with: task-definition: ${{ steps.render.outputs.task-definition }} cluster: ${{ inputs.cluster }} - - - name: Checking that the service answers - if: inputs.smoke-url != '' && !inputs.task-definition-only - env: - URL: ${{ inputs.smoke-url }} - run: | - for attempt in $(seq 1 10); do - if curl -fsS -o /dev/null -m 10 "$URL"; then - echo "$URL answered." - exit 0 - fi - echo "Attempt $attempt: $URL did not answer. Trying again in 6 s." - sleep 6 - done - echo "::error::$URL did not answer after the deploy. Roll back with the Deploy workflow and an older commit." - exit 1 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 3e97672..0000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,212 +0,0 @@ -# Lints and tests a Go repo with its own just recipes, so CI does what a -# person does locally. See the README for an example call. -name: Go - -on: - workflow_call: - inputs: - runner: - description: >- - The runner label. Empty means the RUNNER_LABEL variable, then - ubuntu-latest. Public repos pass ubuntu-latest. - type: string - default: "" - working-directory: - description: The folder with go.mod and the justfile. - type: string - default: "." - go-tools: - description: >- - Tools to install with go install before the lint, separated by - spaces, for example mvdan.cc/gofumpt@v0.12.0. - type: string - default: "" - golangci-lint: - description: >- - Installs this golangci-lint version before the lint, for example - v2.15. The lint command runs it. Empty means no golangci-lint. - type: string - default: "" - lint-command: - type: string - default: just lint - test-command: - type: string - default: just test - clickhouse: - description: >- - Starts a ClickHouse server for the tests. The test command gets - CLICKHOUSE_HOST, CLICKHOUSE_PORT (native protocol), - CLICKHOUSE_HTTP_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD and - CLICKHOUSE_DATABASE. - type: boolean - default: false - clickhouse-image: - type: string - default: clickhouse/clickhouse-server:24.12 - clickhouse-user: - type: string - default: test - clickhouse-password: - type: string - default: test - clickhouse-database: - type: string - default: test - goreleaser-check: - description: >- - Also builds every release binary with goreleaser, without a - release, so a change that breaks a release fails here. - type: boolean - default: false - goprivate: - description: >- - Private modules, comma-separated, for example - github.com/probe-lab/api. Needs the GO_PRIVATE_TOKEN secret. - type: string - default: "" - timeout-minutes: - type: number - default: 30 - secrets: - GO_PRIVATE_TOKEN: - description: A token that can read the modules in goprivate. - required: false - -jobs: - check: - name: Lint and test - runs-on: ${{ inputs.runner || vars.RUNNER_LABEL || 'ubuntu-latest' }} - timeout-minutes: ${{ inputs.timeout-minutes }} - defaults: - run: - working-directory: ${{ inputs.working-directory }} - services: - clickhouse: - # An empty image starts no service. - image: ${{ inputs.clickhouse && inputs.clickhouse-image || '' }} - # Docker picks free host ports, because several jobs can share one - # self-hosted machine. - ports: - - 9000 - - 8123 - env: - CLICKHOUSE_USER: ${{ inputs.clickhouse-user }} - CLICKHOUSE_PASSWORD: ${{ inputs.clickhouse-password }} - CLICKHOUSE_DB: ${{ inputs.clickhouse-database }} - options: >- - --health-cmd "clickhouse-client --query 'SELECT 1'" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checking out the code - uses: actions/checkout@v7 - - # A self-hosted runner keeps the Go caches on its disk, so only - # GitHub-hosted runners use the GitHub cache. - - name: Setting up Go - uses: actions/setup-go@v7 - with: - go-version-file: ${{ inputs.working-directory }}/go.mod - cache: ${{ runner.environment == 'github-hosted' }} - cache-dependency-path: ${{ inputs.working-directory }}/go.sum - - - name: Setting up just - uses: extractions/setup-just@v4 - - # The git settings go into the job's environment, not into a file, so - # the token does not stay on a self-hosted runner after the job. - - name: Giving access to private Go modules - if: inputs.goprivate != '' - env: - MODULES: ${{ inputs.goprivate }} - TOKEN: ${{ secrets.GO_PRIVATE_TOKEN }} - run: | - # Adds to git settings that the environment may already have. - i=${GIT_CONFIG_COUNT:-0} - for module in ${MODULES//,/ }; do - echo "GIT_CONFIG_KEY_$i=url.https://x-access-token:${TOKEN}@${module}.insteadOf" >> "$GITHUB_ENV" - echo "GIT_CONFIG_VALUE_$i=https://${module}" >> "$GITHUB_ENV" - i=$((i + 1)) - done - echo "GIT_CONFIG_COUNT=$i" >> "$GITHUB_ENV" - echo "GOPRIVATE=$MODULES" >> "$GITHUB_ENV" - - - name: Installing Go tools - if: inputs.go-tools != '' - env: - TOOLS: ${{ inputs.go-tools }} - run: | - for tool in $TOOLS; do - go install "$tool" - done - - # Installs the release binary, because golangci-lint advises against - # go install. It only installs; the lint command runs it. - - name: Installing golangci-lint - if: inputs.golangci-lint != '' - uses: golangci/golangci-lint-action@v9 - with: - version: ${{ inputs.golangci-lint }} - install-only: true - working-directory: ${{ inputs.working-directory }} - skip-cache: ${{ runner.environment != 'github-hosted' }} - - - name: Linting - run: ${{ inputs.lint-command }} - - # Runs after a failed lint too, so one run reports both. - - name: Testing - if: ${{ !cancelled() }} - env: - CLICKHOUSE_HOST: localhost - CLICKHOUSE_PORT: ${{ job.services.clickhouse.ports['9000'] }} - CLICKHOUSE_HTTP_PORT: ${{ job.services.clickhouse.ports['8123'] }} - CLICKHOUSE_USER: ${{ inputs.clickhouse-user }} - CLICKHOUSE_PASSWORD: ${{ inputs.clickhouse-password }} - CLICKHOUSE_DATABASE: ${{ inputs.clickhouse-database }} - run: ${{ inputs.test-command }} - - goreleaser: - name: Release build - if: inputs.goreleaser-check - runs-on: ${{ inputs.runner || vars.RUNNER_LABEL || 'ubuntu-latest' }} - timeout-minutes: ${{ inputs.timeout-minutes }} - steps: - # The full history has the tags that go build reads the version from. - - name: Checking out the code - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Setting up Go - uses: actions/setup-go@v7 - with: - go-version-file: ${{ inputs.working-directory }}/go.mod - cache: ${{ runner.environment == 'github-hosted' }} - cache-dependency-path: ${{ inputs.working-directory }}/go.sum - - - name: Giving access to private Go modules - if: inputs.goprivate != '' - env: - MODULES: ${{ inputs.goprivate }} - TOKEN: ${{ secrets.GO_PRIVATE_TOKEN }} - run: | - # Adds to git settings that the environment may already have. - i=${GIT_CONFIG_COUNT:-0} - for module in ${MODULES//,/ }; do - echo "GIT_CONFIG_KEY_$i=url.https://x-access-token:${TOKEN}@${module}.insteadOf" >> "$GITHUB_ENV" - echo "GIT_CONFIG_VALUE_$i=https://${module}" >> "$GITHUB_ENV" - i=$((i + 1)) - done - echo "GIT_CONFIG_COUNT=$i" >> "$GITHUB_ENV" - echo "GOPRIVATE=$MODULES" >> "$GITHUB_ENV" - - - name: Building the release binaries - uses: goreleaser/goreleaser-action@v7 - with: - distribution: goreleaser - version: "~> v2" - args: build --snapshot --clean - workdir: ${{ inputs.working-directory }} diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 2e90da7..cb0c6e5 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -1,9 +1,14 @@ -# Builds a Docker image. With push, it pushes the image to ECR as -# -sha; a pull request also gets -pr. -# Without push, it only checks that the image builds. See the README for an -# example call. +# Builds a Docker image and pushes it to ECR as -sha; a pull +# request also gets -pr. It pushes the image of a merge, of a +# manual run, and of a pull request with the label push-image. Any other pull +# request only builds the image, to check that it builds. See the README for +# an example call. # -# The caller grants id-token: write when push is true. +# A pull request with other labels does not run it again: the caller lists +# `labeled` in its pull_request types, and this workflow skips a labeled +# event of any other label. The image goes to the ProbeLab ECR repository, and +# the build runs on the RUNNER_LABEL runner when the repo has that variable. +# The caller grants id-token: write. name: Image on: @@ -14,20 +19,12 @@ on: type: string required: true push: - description: Pushes the image to ECR. Without it, the image is only built. + description: >- + Pushes whatever the event is. Without it, the rules above decide. type: boolean default: false - runner: - description: >- - The runner label. Empty means the RUNNER_LABEL variable, then - ubuntu-latest. Public repos pass ubuntu-latest. - type: string - default: "" - context: - type: string - default: "." dockerfile: - description: The Dockerfile, as a path from the repo root. + description: The Dockerfile, as a path from the repo root. The build context is the repo root. type: string default: Dockerfile build-args: @@ -37,33 +34,12 @@ on: platforms: type: string default: linux/amd64 - fetch-depth: + commit: description: >- - The git history to check out. 0 means all of it, which a build - needs when go build reads the version from the tags. - type: number - default: 1 - aws-region: - type: string - default: us-east-1 - aws-role: + The commit to build, as a SHA, a branch or a tag. Empty means the + commit of the event, or the head commit of a pull request. type: string - default: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role - ecr-registry: - type: string - default: 019120760881.dkr.ecr.us-east-1.amazonaws.com - ecr-repository: - type: string - default: probelab - timeout-minutes: - type: number - default: 45 - secrets: - GO_PRIVATE_TOKEN: - description: >- - Given to the build as the secret github_token, for Dockerfiles - that download private Go modules. - required: false + default: "" outputs: image: description: The image with its commit tag, for example 0191.../probelab:atlas-sha1a2b3c4. @@ -72,31 +48,45 @@ on: description: The commit tag, for example atlas-sha1a2b3c4. value: ${{ jobs.image.outputs.tag }} +env: + AWS_REGION: us-east-1 + AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role + ECR_REGISTRY: 019120760881.dkr.ecr.us-east-1.amazonaws.com + ECR_REPOSITORY: probelab + jobs: image: name: Image - runs-on: ${{ inputs.runner || vars.RUNNER_LABEL || 'ubuntu-latest' }} - timeout-minutes: ${{ inputs.timeout-minutes }} + # The label push-image on a pull request is the only other label that + # starts a run. + if: github.event.action != 'labeled' || github.event.label.name == 'push-image' + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-latest' }} + timeout-minutes: 45 + env: + PUSH: ${{ inputs.push || github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'push-image') }} outputs: image: ${{ steps.tags.outputs.image }} tag: ${{ steps.tags.outputs.tag }} steps: # A pull request builds its head commit, not the merge commit, so the - # tag names a commit that exists in the repo. + # tag names a commit that exists in the repo. The full history has the + # tags that go build reads the version from. - name: Checking out the code uses: actions/checkout@v7 with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - fetch-depth: ${{ inputs.fetch-depth }} + ref: ${{ inputs.commit || github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 - name: Computing the image tags id: tags env: SERVICE: ${{ inputs.service }} - IMAGE: ${{ inputs.ecr-registry }}/${{ inputs.ecr-repository }} - SHA: ${{ github.event.pull_request.head.sha || github.sha }} + IMAGE: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }} PR: ${{ github.event.pull_request.number }} run: | + # The checked-out commit, so a branch name or a short SHA in the + # commit input gives the same tag as the full SHA. + SHA=$(git rev-parse HEAD) tag="$SERVICE-sha${SHA::7}" tags="$IMAGE:$tag" if [[ -n "$PR" ]]; then @@ -109,23 +99,21 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Configuring AWS credentials - if: inputs.push + if: env.PUSH == 'true' uses: aws-actions/configure-aws-credentials@v6 with: - aws-region: ${{ inputs.aws-region }} - role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ env.AWS_ROLE }} - name: Logging in to Amazon ECR - if: inputs.push + if: env.PUSH == 'true' uses: aws-actions/amazon-ecr-login@v2 - # A commit is built once. A rerun of a merged commit skips the build. - # A pull request always builds, so its pr tag follows the latest push. - name: Checking if the image exists id: exists - if: inputs.push && github.event_name != 'pull_request' + if: env.PUSH == 'true' && github.event_name != 'pull_request' env: - REPOSITORY: ${{ inputs.ecr-repository }} + REPOSITORY: ${{ env.ECR_REPOSITORY }} TAG: ${{ steps.tags.outputs.tag }} run: | if aws ecr describe-images --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" > /dev/null 2>&1; then @@ -147,11 +135,9 @@ jobs: if: steps.exists.outputs.exists != 'true' uses: docker/build-push-action@v7 with: - context: ${{ inputs.context }} + context: . file: ${{ inputs.dockerfile }} platforms: ${{ inputs.platforms }} build-args: ${{ inputs.build-args }} - secrets: | - github_token=${{ secrets.GO_PRIVATE_TOKEN }} - push: ${{ inputs.push }} + push: ${{ env.PUSH == 'true' }} tags: ${{ steps.tags.outputs.tags }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index 04acffa..0000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,125 +0,0 @@ -# Lints and tests a Python repo that uses uv and just, so CI does what a -# person does locally. See the README for an example call. -name: Python - -on: - workflow_call: - inputs: - runner: - description: >- - The runner label. Empty means the RUNNER_LABEL variable, then - ubuntu-latest. Public repos pass ubuntu-latest. - type: string - default: "" - working-directory: - description: The folder with pyproject.toml, .python-version and the justfile. - type: string - default: "." - lfs: - description: Checks out Git LFS files too. - type: boolean - default: false - uv-version: - description: The uv version. Empty means the version that pyproject.toml asks for, or the newest. - type: string - default: "" - sync-command: - type: string - default: uv sync --locked - lint-command: - type: string - default: just lint - test-command: - type: string - default: just test - clickhouse: - description: >- - Starts a ClickHouse server for the tests. The test command gets - CLICKHOUSE_HOST, CLICKHOUSE_PORT (native protocol), - CLICKHOUSE_HTTP_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD and - CLICKHOUSE_DATABASE. - type: boolean - default: false - clickhouse-image: - type: string - default: clickhouse/clickhouse-server:24.12 - clickhouse-user: - type: string - default: test - clickhouse-password: - type: string - default: test - clickhouse-database: - type: string - default: test - timeout-minutes: - type: number - default: 30 - -jobs: - check: - name: Lint and test - runs-on: ${{ inputs.runner || vars.RUNNER_LABEL || 'ubuntu-latest' }} - timeout-minutes: ${{ inputs.timeout-minutes }} - defaults: - run: - working-directory: ${{ inputs.working-directory }} - services: - clickhouse: - # An empty image starts no service. - image: ${{ inputs.clickhouse && inputs.clickhouse-image || '' }} - # Docker picks free host ports, because several jobs can share one - # self-hosted machine. - ports: - - 9000 - - 8123 - env: - CLICKHOUSE_USER: ${{ inputs.clickhouse-user }} - CLICKHOUSE_PASSWORD: ${{ inputs.clickhouse-password }} - CLICKHOUSE_DB: ${{ inputs.clickhouse-database }} - options: >- - --health-cmd "clickhouse-client --query 'SELECT 1'" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checking out the code - uses: actions/checkout@v7 - with: - lfs: ${{ inputs.lfs }} - - # A self-hosted runner keeps the uv cache on its disk, so only - # GitHub-hosted runners use the GitHub cache. - - name: Setting up uv - uses: astral-sh/setup-uv@v7 - with: - version: ${{ inputs.uv-version }} - enable-cache: ${{ runner.environment == 'github-hosted' }} - working-directory: ${{ inputs.working-directory }} - - - name: Setting up Python - uses: actions/setup-python@v7 - with: - python-version-file: ${{ inputs.working-directory }}/.python-version - - - name: Setting up just - uses: extractions/setup-just@v4 - - - name: Installing the dependencies - id: sync - run: ${{ inputs.sync-command }} - - - name: Linting - run: ${{ inputs.lint-command }} - - # Runs after a failed lint too, so one run reports both. - - name: Testing - if: ${{ !cancelled() && steps.sync.outcome == 'success' }} - env: - CLICKHOUSE_HOST: localhost - CLICKHOUSE_PORT: ${{ job.services.clickhouse.ports['9000'] }} - CLICKHOUSE_HTTP_PORT: ${{ job.services.clickhouse.ports['8123'] }} - CLICKHOUSE_USER: ${{ inputs.clickhouse-user }} - CLICKHOUSE_PASSWORD: ${{ inputs.clickhouse-password }} - CLICKHOUSE_DATABASE: ${{ inputs.clickhouse-database }} - run: ${{ inputs.test-command }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 906eb28..7292ec1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,14 @@ # Publishes a release for a version tag such as v1.2.3 or v1.2.3-rc.1: the -# GitHub release with notes from CHANGELOG.md, optional goreleaser binaries, -# and optional images. A tag with -rc.N becomes a pre-release. See the README -# for an example call. +# GitHub release with notes from the "## [1.2.3]" section of CHANGELOG.md, and +# an optional ECR image. A tag with -rc.N becomes a pre-release. It fails +# unless the tagged commit passed the check "Checking the code" on the default +# branch, so a repo does not run its checks again. See the README for an example call. # -# Releases hold write credentials, so every job runs on a GitHub-hosted -# runner. The caller grants contents: write, and id-token: write when it -# pushes images to ECR. +# The repo's .goreleaser.yaml, if there is one, builds and attaches the +# binaries; a release candidate is a pre-release when it sets +# release.prerelease to auto. Releases hold write credentials, so every job +# runs on a GitHub-hosted runner. The caller grants contents: write, +# checks: read, and id-token: write when it pushes an image to ECR. name: Release on: @@ -15,54 +18,21 @@ on: description: The tag to release, for example v1.2.3. type: string required: true - changelog: - description: >- - Takes the release notes from the "## [1.2.3]" section of - CHANGELOG.md and fails without one. A release candidate uses the - section of the version it leads to. Without it, GitHub writes the - notes from the merged pull requests. - type: boolean - default: true - goreleaser: - description: Builds and attaches the binaries with the repo's .goreleaser.yaml. - type: boolean - default: false ecr-service: description: >- Pushes an image to ECR as -1.2.3, and as - -latest for a final release. Empty means no ECR image. - type: string - default: "" - dockerhub-repository: - description: >- - Pushes an image to Docker Hub as :v1.2.3, and as - :latest for a final release. Empty means no Docker Hub - image. + -latest for a final release. Empty means no image. type: string default: "" - dockerfile: - type: string - default: Dockerfile platforms: type: string default: linux/amd64,linux/arm64 - aws-region: - type: string - default: us-east-1 - aws-role: - type: string - default: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role - ecr-registry: - type: string - default: 019120760881.dkr.ecr.us-east-1.amazonaws.com - ecr-repository: - type: string - default: probelab - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false + +env: + AWS_REGION: us-east-1 + AWS_ROLE: arn:aws:iam::019120760881:role/prod-use1-github-oidc-role + ECR_REGISTRY: 019120760881.dkr.ecr.us-east-1.amazonaws.com + ECR_REPOSITORY: probelab jobs: prepare: @@ -88,6 +58,23 @@ jobs: echo "prerelease=false" >> "$GITHUB_OUTPUT" fi + # The checks ran when the commit was merged. A tag on a commit that + # never passed them is a mistake, so the release stops here. + - name: Checking that CI passed for the tagged commit + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + sha=$(gh api "repos/$REPO/commits/$TAG" --jq .sha) + result=$(gh api "repos/$REPO/commits/$sha/check-runs?check_name=Checking%20the%20code" \ + --jq '[.check_runs[] | select(.conclusion == "success")] | length') + if [[ "$result" == "0" ]]; then + echo "::error::The commit ${sha::7} has no passing 'Checking the code' check. Merge it through a pull request, wait for CI, and tag again." + exit 1 + fi + github-release: name: GitHub release needs: prepare @@ -101,7 +88,6 @@ jobs: fetch-depth: 0 - name: Taking the notes from CHANGELOG.md - if: inputs.changelog env: VERSION: ${{ needs.prepare.outputs.version }} run: | @@ -119,47 +105,46 @@ jobs: fi printf '%s\n' "$notes" > "$RUNNER_TEMP/notes.md" + - name: Looking for a goreleaser configuration + id: goreleaser + run: | + if compgen -G ".goreleaser.y*ml" > /dev/null; then + echo "found=true" >> "$GITHUB_OUTPUT" + fi + - name: Setting up Go - if: inputs.goreleaser + if: steps.goreleaser.outputs.found == 'true' uses: actions/setup-go@v7 with: go-version-file: go.mod - # goreleaser makes a pre-release for a -rc tag when the repo's - # .goreleaser.yaml sets release.prerelease to auto. - name: Releasing with goreleaser - if: inputs.goreleaser + if: steps.goreleaser.outputs.found == 'true' uses: goreleaser/goreleaser-action@v7 with: distribution: goreleaser version: "~> v2" - args: release --clean ${{ inputs.changelog && format('--release-notes {0}/notes.md', runner.temp) || '' }} + args: release --clean --release-notes ${{ runner.temp }}/notes.md env: GITHUB_TOKEN: ${{ github.token }} - name: Making the GitHub release - if: ${{ !inputs.goreleaser }} + if: steps.goreleaser.outputs.found != 'true' env: GH_TOKEN: ${{ github.token }} TAG: ${{ inputs.tag }} PRERELEASE: ${{ needs.prepare.outputs.prerelease }} - CHANGELOG: ${{ inputs.changelog }} run: | - args=(--verify-tag --title "$TAG") + args=(--verify-tag --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md") if [[ "$PRERELEASE" == "true" ]]; then args+=(--prerelease) fi - if [[ "$CHANGELOG" == "true" ]]; then - args+=(--notes-file "$RUNNER_TEMP/notes.md") - else - args+=(--generate-notes) - fi gh release create "$TAG" "${args[@]}" - images: - name: Images + image: + name: ECR image needs: prepare - if: inputs.ecr-service != '' || inputs.dockerhub-repository != '' + if: inputs.ecr-service != '' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -173,53 +158,36 @@ jobs: - name: Computing the image tags id: tags env: - TAG: ${{ inputs.tag }} VERSION: ${{ needs.prepare.outputs.version }} PRERELEASE: ${{ needs.prepare.outputs.prerelease }} - ECR_SERVICE: ${{ inputs.ecr-service }} - ECR_IMAGE: ${{ inputs.ecr-registry }}/${{ inputs.ecr-repository }} - DOCKERHUB: ${{ inputs.dockerhub-repository }} + IMAGE: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }} + SERVICE: ${{ inputs.ecr-service }} run: | - tags=() - if [[ -n "$ECR_SERVICE" ]]; then - tags+=("$ECR_IMAGE:$ECR_SERVICE-$VERSION") - [[ "$PRERELEASE" == "true" ]] || tags+=("$ECR_IMAGE:$ECR_SERVICE-latest") + tags="$IMAGE:$SERVICE-$VERSION" + if [[ "$PRERELEASE" != "true" ]]; then + tags="$tags,$IMAGE:$SERVICE-latest" fi - if [[ -n "$DOCKERHUB" ]]; then - tags+=("$DOCKERHUB:$TAG") - [[ "$PRERELEASE" == "true" ]] || tags+=("$DOCKERHUB:latest") - fi - (IFS=,; echo "tags=${tags[*]}") >> "$GITHUB_OUTPUT" + echo "tags=$tags" >> "$GITHUB_OUTPUT" - name: Configuring AWS credentials - if: inputs.ecr-service != '' uses: aws-actions/configure-aws-credentials@v6 with: - aws-region: ${{ inputs.aws-region }} - role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ env.AWS_ROLE }} - name: Logging in to Amazon ECR - if: inputs.ecr-service != '' uses: aws-actions/amazon-ecr-login@v2 - - name: Logging in to Docker Hub - if: inputs.dockerhub-repository != '' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Setting up QEMU uses: docker/setup-qemu-action@v4 - name: Setting up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Building and pushing the images + - name: Building and pushing the image uses: docker/build-push-action@v7 with: context: . - file: ${{ inputs.dockerfile }} platforms: ${{ inputs.platforms }} push: true tags: ${{ steps.tags.outputs.tags }} diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml deleted file mode 100644 index 481ed9c..0000000 --- a/.github/workflows/terraform.yml +++ /dev/null @@ -1,198 +0,0 @@ -# Runs Terraform in CI. mode: plan checks the code, plans, and posts the plan -# on the pull request. mode: apply plans and applies in the given GitHub -# environment; a person starts it, a merge never does. See the README for an -# example call. -# -# Every secret whose name starts with TF_VAR_ becomes a Terraform variable: -# the secret TF_VAR_CMI_RDS_PASSWORD sets the variable cmi_rds_password. Call -# this workflow with secrets: inherit. -# -# The caller grants contents: read and id-token: write, and pull-requests: -# write for a plan on a pull request. -name: Terraform - -on: - workflow_call: - inputs: - mode: - description: plan or apply. - type: string - required: true - working-directory: - type: string - default: tf - terraform-version: - type: string - default: 1.14.8 - aws-region: - type: string - default: us-east-1 - aws-role: - description: The role to assume. Use a read-only role for plans on pull requests. - type: string - required: true - environment: - description: The GitHub environment of an apply. - type: string - default: production - gcp-service-account: - description: A GCP service account to sign in as, through the GCP_WORKLOAD_POOL secret. Empty means no GCP. - type: string - default: "" - secrets: - GCP_WORKLOAD_POOL: - required: false - -jobs: - plan: - name: Plan - if: inputs.mode == 'plan' - runs-on: ubuntu-latest - timeout-minutes: 30 - defaults: - run: - working-directory: ${{ inputs.working-directory }} - steps: - - name: Checking out the code - uses: actions/checkout@v7 - - # A secret value can hold any character, so each one is written with - # its own random delimiter. - - name: Turning the TF_VAR_ secrets into variables - env: - SECRETS: ${{ toJSON(secrets) }} - run: | - while IFS= read -r key; do - name="TF_VAR_$(tr '[:upper:]' '[:lower:]' <<< "${key#TF_VAR_}")" - delimiter="EOF_$(openssl rand -hex 16)" - { - echo "$name<<$delimiter" - jq -r --arg k "$key" '.[$k]' <<< "$SECRETS" - echo "$delimiter" - } >> "$GITHUB_ENV" - done < <(jq -r 'keys[] | select(startswith("TF_VAR_"))' <<< "$SECRETS") - - - name: Configuring AWS credentials - uses: aws-actions/configure-aws-credentials@v6 - with: - aws-region: ${{ inputs.aws-region }} - role-to-assume: ${{ inputs.aws-role }} - - - name: Configuring GCP credentials - if: inputs.gcp-service-account != '' - uses: google-github-actions/auth@v3 - with: - service_account: ${{ inputs.gcp-service-account }} - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_POOL }} - - - name: Setting up Terraform - uses: hashicorp/setup-terraform@v4 - with: - terraform_version: ${{ inputs.terraform-version }} - terraform_wrapper: false - - - name: Initializing - run: terraform init -input=false - - - name: Checking the format - run: terraform fmt -check - - - name: Validating - run: terraform validate - - - name: Planning - run: | - terraform plan -input=false -no-color -lock-timeout=5m -out=plan.out - terraform show -no-color plan.out > plan.txt - - # GitHub cuts a comment at 65536 characters, so a long plan is cut and - # the comment says where the whole plan is. - - name: Posting the plan on the pull request - if: github.event_name == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - PR: ${{ github.event.pull_request.number }} - RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - summary=$(grep -E '^(Plan:|No changes\.)' plan.txt | tail -1) - { - echo "#### Terraform plan: ${summary:-see the run}" - echo - echo "
Plan" - echo - echo '```' - head -c 60000 plan.txt - echo '```' - echo - echo "
" - echo - if [[ $(wc -c < plan.txt) -gt 60000 ]]; then - echo "The plan is longer than this comment. The whole plan is in [the run]($RUN)." - fi - } > comment.md - gh pr comment "$PR" --body-file comment.md - - apply: - name: Apply - if: inputs.mode == 'apply' - runs-on: ubuntu-latest - timeout-minutes: 60 - environment: ${{ inputs.environment }} - concurrency: - group: terraform-apply-${{ github.repository }}-${{ inputs.working-directory }} - cancel-in-progress: false - defaults: - run: - working-directory: ${{ inputs.working-directory }} - steps: - - name: Checking out the code - uses: actions/checkout@v7 - - - name: Turning the TF_VAR_ secrets into variables - env: - SECRETS: ${{ toJSON(secrets) }} - run: | - while IFS= read -r key; do - name="TF_VAR_$(tr '[:upper:]' '[:lower:]' <<< "${key#TF_VAR_}")" - delimiter="EOF_$(openssl rand -hex 16)" - { - echo "$name<<$delimiter" - jq -r --arg k "$key" '.[$k]' <<< "$SECRETS" - echo "$delimiter" - } >> "$GITHUB_ENV" - done < <(jq -r 'keys[] | select(startswith("TF_VAR_"))' <<< "$SECRETS") - - - name: Configuring AWS credentials - uses: aws-actions/configure-aws-credentials@v6 - with: - aws-region: ${{ inputs.aws-region }} - role-to-assume: ${{ inputs.aws-role }} - - - name: Configuring GCP credentials - if: inputs.gcp-service-account != '' - uses: google-github-actions/auth@v3 - with: - service_account: ${{ inputs.gcp-service-account }} - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_POOL }} - - - name: Setting up Terraform - uses: hashicorp/setup-terraform@v4 - with: - terraform_version: ${{ inputs.terraform-version }} - terraform_wrapper: false - - - name: Initializing - run: terraform init -input=false - - - name: Planning - run: | - terraform plan -input=false -no-color -lock-timeout=5m -out=plan.out - { - echo '### Terraform plan' - echo '```' - terraform show -no-color plan.out | head -c 900000 - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - - name: Applying - run: terraform apply -input=false -no-color -lock-timeout=5m plan.out diff --git a/AGENTS.md b/AGENTS.md index e19879f..551223c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,27 +2,29 @@ ProbeLab repository. Follow the `probelab` skill: it holds the company rules. Load it before you start any task here. -Org-level repo of `probe-lab`. It holds the shared reusable workflows that every ProbeLab repo calls at `@v1`, the workflow templates, and the default community health files (security policy, pull request template, org profile). +Org-level repo of `probe-lab`. It holds the shared reusable workflows that ProbeLab repos call at `@v2`, the workflow templates, and the default community health files (security policy, pull request template, org profile). ## Layout -- `.github/workflows/` — shared workflows (`go.yml`, `python.yml`, `image.yml`, `deploy-ecs.yml`, `release.yml`, `terraform.yml`) and this repo's own `ci.yml` +- `.github/workflows/` — shared workflows (`image.yml`, `deploy-ecs.yml`, `release.yml`) and this repo's own `ci.yml` - `workflow-templates/` — starter workflows offered under Actions, New workflow -- `tests/go/`, `tests/python/` — small fixture projects that CI runs the shared workflows against +- `scripts/` — `lint.sh`, which `just lint` and CI run +- `tests/image/` — the small Dockerfile that CI builds with `image.yml` - `profile/` — public org page - `PULL_REQUEST_TEMPLATE.md`, `SECURITY.md`, `CODEOWNERS` — org defaults - `CHANGELOG.md` — changes to the shared workflows, by version ## Commands -- `just lint` runs actionlint on the workflows (needs `actionlint` on the PATH); `just ci` runs the checks. -- CI also lints the templates and runs the shared workflows against `tests/`. `deploy-ecs.yml`, `release.yml` and `terraform.yml` need AWS, so only actionlint checks them. +- `just lint` runs `scripts/lint.sh`: actionlint on the workflows and on the templates (downloads `actionlint` into `~/.cache/actionlint` if it is not on the PATH); `just ci` runs the checks. CI runs `just lint` too. +- CI also builds an image with `image.yml`. `deploy-ecs.yml` and `release.yml` need AWS, so only actionlint checks them. ## Conventions -- Callers pin `@v1`. Do not rename or remove an input or output, or change behavior, in a way that breaks a caller: that is a new major version. -- Do not move or retag `v1` as part of a normal change. The README, "Change this repo", has the release steps. +- Callers pin `@v2`. Do not rename or remove an input or output, or change behavior, in a way that breaks a caller: that is a new major version. +- Do not move or retag `v2` as part of a normal change. The README, "Change this repo", has the release steps. - Write changes under a new `## [X.Y.Z]` section in `CHANGELOG.md`. +- Shared workflows hold production credentials or a naming contract only. A need of one repo goes in that repo's `ci.yml` or `justfile`, not in an input here. - A change to a template goes with the matching change in the README examples. - Change `PULL_REQUEST_TEMPLATE.md` together with the template in the `probelab-git` skill. diff --git a/CHANGELOG.md b/CHANGELOG.md index beb3ca6..3197baa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,49 @@ major version such as `@v1`. The format follows ## [Unreleased] +Version 2 moves the special cases of single repos out of the shared +workflows. A caller of version 1 keeps working at `@v1`. + ### Removed -- `deploy-ecs.yml`: the `SLACK_WEBHOOK_URL` secret and the Slack message after - a deploy. A caller that passes `SLACK_WEBHOOK_URL:` under `secrets:` now - fails at startup. Remove that line. Callers with `secrets: inherit` are not - affected. +- `go.yml`, `python.yml`: a repo now spells out a `check` job that runs + `just ci`, from the templates `go-service-ci`, `go-library-ci` and + `python-service-ci`. Tool versions move into the repo's `justfile`. +- `terraform.yml`: no repo called it. +- `image.yml`: the inputs `runner`, `context`, `fetch-depth`, `aws-*`, + `ecr-*`, `timeout-minutes` and the secret `GO_PRIVATE_TOKEN`. The runner is + the `RUNNER_LABEL` variable, the build context is the repo root, and the + checkout always has the full history. +- `deploy-ecs.yml`: the inputs `pr-label`, `smoke-url`, `environment`, + `wait-minutes`, `image-wait-minutes`, `aws-*` and `ecr-*`, and the + `SLACK_WEBHOOK_URL` secret with the Slack message after a deploy. +- `release.yml`: the inputs `changelog`, `goreleaser`, `dockerfile`, + `dockerhub-repository`, `aws-*` and `ecr-*`, and the Docker Hub secrets. + +### Changed + +- `deploy-ecs.yml`: the `deploy` label check applies by itself when the CI run + of a merge (a `push` event) calls it. The Deploy button does not check the + label. The workflow no longer waits for an image: the `deploy` job of + `ci.yml` runs after the `image` job of the same run, on the merge commit. A + repo has one `deploy.yml` with `workflow_dispatch` and `workflow_call`. +- `release.yml`: fails unless the tagged commit has a passing + `Checking the code` check. The caller grants `checks: read`. The workflow uses goreleaser when the repo + has a `.goreleaser.yaml`, and always takes the notes from `CHANGELOG.md`. +- The workflow templates call `@v2`. The service templates have a `deploy` + job, and `service-deploy` is the button and the target of that job. +- The CI of this repo and the templates have no `ci-ok` job. The required + check is `Checking the code`, and for a service the image job. + +### Added + +- `image.yml`: pushes the image of a pull request that has the label + `push-image`. The `push` input defaults to false and forces the push; without it + the workflow pushes a merge, a manual run, or a pull request with that label. The + service templates list `labeled` in their pull request types. +- `image.yml`: the `commit` input, to build a commit other than the one of + the event. The Deploy button of a repo uses it. +- The template `python-service-ci`. ## [1.1.0] diff --git a/README.md b/README.md index cac11c1..ae0069e 100644 --- a/README.md +++ b/README.md @@ -1,332 +1,138 @@ # ProbeLab's shared GitHub files -This repo holds the GitHub Actions workflows that every ProbeLab repo calls, -the templates for new workflows, and the default files for repos that have -none of their own. The plan behind it is the Notion page "ProbeLab CI, -release and deploy strategy". +This repo holds the GitHub Actions workflows that ProbeLab repos share, the +templates for new workflows, and the default files for repos that have none +of their own. -In short: a pull request runs the checks, a merge builds one image, and a +In short: a pull request runs `just ci`, a merge builds one image, and a person deploys that image with one click. A merge deploys only when its pull request has the label `deploy`. -## What is here +## What is shared, and what is not + +A workflow is shared only when it holds production credentials or sets a +naming contract that all repos must follow. The steps of a check are not +shared: every repo spells them out from a template, because a check is +`setup`, `just`, and `just ci`. The special needs of one repo, such as a +ClickHouse container for the tests, belong in that repo's own `ci.yml`. | Path | What it is | | --- | --- | -| `.github/workflows/go.yml` | Lints and tests a Go repo with its `just` recipes, with an optional ClickHouse and an optional goreleaser check. | -| `.github/workflows/python.yml` | Lints and tests a Python repo with `uv` and `just`, with an optional ClickHouse. | | `.github/workflows/image.yml` | Builds a Docker image, and pushes it to ECR as `-sha` when asked. | -| `.github/workflows/deploy-ecs.yml` | Deploys the image of one commit to one ECS service. Every Deploy button calls it. | -| `.github/workflows/release.yml` | Publishes a version tag: the GitHub release, and optional binaries and images. | -| `.github/workflows/terraform.yml` | Plans Terraform on a pull request, and applies it when a person starts it. | +| `.github/workflows/deploy-ecs.yml` | Deploys the image of one commit to one ECS service. | +| `.github/workflows/release.yml` | Publishes a version tag: the GitHub release with the notes from `CHANGELOG.md`, goreleaser binaries when the repo has a `.goreleaser.yaml`, and an optional ECR image. | | `workflow-templates/` | Starter files that every org repo offers under Actions, New workflow. | | `PULL_REQUEST_TEMPLATE.md`, `SECURITY.md` | Defaults for every org repo that has no file of its own. | | `profile/README.md` | The public page of the org. | | `.github/workflows/ci.yml`, `tests/` | The CI of this repo. | -Each workflow lists all of its inputs at the top of its file. - -## Use it in a repo - -Repos call the workflows with a major version, `@v1`. A service repo has 2 -files: `ci.yml` checks every pull request and builds the image of every -merge, and `deploy.yml` is the Deploy button. `ci.yml` also calls -`deploy.yml` after a merge with the label `deploy`. Start from the templates: in -the repo, open Actions, then New workflow, and pick a ProbeLab template. - -### A Go service - -```yaml -jobs: - go: - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - - image: - needs: go - permissions: - contents: read - id-token: write - uses: probe-lab/.github/.github/workflows/image.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - service: boomo - push: ${{ github.event_name == 'push' }} -``` - -The whole file, with the `ci-ok` job that branch rules require, is -`workflow-templates/go-service-ci.yml`. - -A repo whose `just lint` runs golangci-lint sets `golangci-lint` to the -version to install, for example `golangci-lint: v2.15`. Other tools come -from `go-tools`, which installs them with `go install`. - -A repo with more needs sets more inputs. nebula needs gofumpt, a ClickHouse -with its own user, its own name for the port, and a check of the release -build: - -```yaml - go: - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - go-tools: mvdan.cc/gofumpt@v0.12.0 - clickhouse: true - clickhouse-user: nebula_test - clickhouse-password: password_test - clickhouse-database: nebula_test - test-command: NEBULA_TEST_CLICKHOUSE_PORT=$CLICKHOUSE_PORT just test-plain - goreleaser-check: true -``` - -A repo that uses a private module, such as boomo with the api module, gives a -token to both workflows. `image.yml` passes it to the Docker build as the -secret `github_token`: - -```yaml - go: - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - goprivate: github.com/probe-lab/api - secrets: - GO_PRIVATE_TOKEN: ${{ secrets.REPO_PROBE_LAB_API_READ }} -``` +Each shared workflow lists all of its inputs at the top of its file. -In the Dockerfile, read the token only in the `RUN --mount=type=secret` step -that downloads the modules. Self-hosted runners keep the build cache, so a -token that a step writes to a file, such as a git config, stays on the -machine. +## The files of a repo -### A Python service +| File | Content | +| --- | --- | +| `ci.yml` | The `check` job, and the `image` job for a service. | +| `deploy.yml` | Only in a service that deploys. The Deploy button, and the deploy after a merge. | +| `release.yml` | Only in a repo with releases. | +| `.github/dependabot.yml` | Keeps the action versions of the repo current. | -atlas keeps its database backups in Git LFS and has its own sync recipe: +Start from the templates: in the repo, open Actions, then New workflow, and +pick a ProbeLab template. The templates are `go-library-ci`, +`go-service-ci`, `python-service-ci`, `service-deploy` and `release`. -```yaml - python: - uses: probe-lab/.github/.github/workflows/python.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - lfs: true - sync-command: just sync - lint-command: just lint && just format-check -``` +### The `check` job -### Several images in one repo +The job checks out the code, sets up the language, `just`, and runs +`just ci`. The pinned versions of tools such as `gofumpt` or `golangci-lint` +live in the `justfile`, for example `go run mvdan.cc/gofumpt@v0.12.0 -l .`, +so a person and CI run the same thing. -data-platform builds 3 images, one for each Dagster component: +A repo adds what only it needs to its own job: -```yaml - image: - needs: python - strategy: - matrix: - component: [daemon, usercode, webserver] - permissions: - contents: read - id-token: write - uses: probe-lab/.github/.github/workflows/image.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - service: dagster-${{ matrix.component }} - dockerfile: Dockerfile.${{ matrix.component }} - push: ${{ github.event_name == 'push' }} -``` +- A service container: nebula starts ClickHouse and priv-hermes starts + LocalStack, and the tests read the mapped port from the environment. +- `lfs: true` on the checkout: atlas keeps database backups in Git LFS. +- A step that only this repo has, such as the release build of nebula. -### Several services from one Dockerfile +When `just test` starts its own services on a laptop, the repo adds a +`test-plain` recipe that expects the services, and CI runs +`just lint test-plain`. -api builds 5 services from one Dockerfile, and the build argument `SERVICE` -selects the service. The tags keep api's `service` names, because -the nebula and ants-watch repos push `nebula-` and `ants-` tags to the same -ECR repository. Every merge builds all 5 images, so every service can deploy -every commit: +### A service image ```yaml - go: - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - golangci-lint: v2.15 - go-tools: github.com/bufbuild/buf/cmd/buf@v1.75.0 - test-command: just test && just test-integration - image: - needs: go - strategy: - matrix: - service: [api, ants, geoip, nebula, probe] + needs: check permissions: contents: read id-token: write - uses: probe-lab/.github/.github/workflows/image.yml@v1 + uses: probe-lab/.github/.github/workflows/image.yml@v2 with: - runner: ${{ vars.RUNNER_LABEL }} - service: ${{ matrix.service }}service - build-args: SERVICE=${{ matrix.service }} + service: boomo push: ${{ github.event_name == 'push' }} ``` -Its Deploy button asks for the service: - -```yaml -on: - workflow_dispatch: - inputs: - service: - type: choice - options: [api, ants, geoip, nebula, probe] - commit: - description: The commit to deploy. Empty means the newest on the default branch. - type: string - required: false - -jobs: - deploy: - uses: probe-lab/.github/.github/workflows/deploy-ecs.yml@v1 - with: - service: ${{ inputs.service }}service - commit: ${{ inputs.commit }} - cluster: prod-use1-web-ecs-cluster - ecs-service: prod-use1-web-${{ inputs.service }}-service - task-definition: prod-use1-web-${{ inputs.service }}-task - container: prod-use1-web-${{ inputs.service }} - secrets: inherit -``` - -### The Deploy button - -`deploy.yml` deploys the newest commit on the default branch, or the commit -that the person names. It stops if the commit is not on the default branch. -If the image of the commit is not in ECR yet, because CI still builds it, the -deploy waits up to 10 minutes (`image-wait-minutes`) and then stops. So a -person can press Deploy right after a merge. The template is -`workflow-templates/service-deploy.yml`. A repo with several services uses a -matrix; each service gets its own rollout: - -```yaml -jobs: - deploy: - strategy: - matrix: - project: [celestia, agntcy] - uses: probe-lab/.github/.github/workflows/deploy-ecs.yml@v1 - with: - service: ants - commit: ${{ inputs.commit }} - cluster: prod-use1-cmi-ecs-cluster - ecs-service: prod-use1-cmi-ants-${{ matrix.project }}-watch - task-definition: prod-use1-cmi-ants-${{ matrix.project }}-watch-task - container: prod-use1-cmi-ants-${{ matrix.project }}-watch - secrets: inherit -``` - -Other inputs: `container-env` sets environment variables in the container, -`smoke-url` checks a URL after the deploy, and `task-definition-only` -registers a task definition without a service deploy. The workflow takes no -secrets. - -To deploy from a terminal, add this recipe to the repo's justfile: - -```just -# deploys a commit, by default the newest on the default branch -deploy commit="": - gh workflow run deploy.yml -f commit={{commit}} -``` - -To roll back, deploy an older commit. +The other inputs are `dockerfile`, `build-args` and `platforms`. The build +context is the repo root. -### Deploy a merge with a label +Several images from one repo use a matrix. data-platform builds one image for +each Dagster component with `dockerfile: Dockerfile.${{ matrix.component }}`. +api builds 5 services from one Dockerfile with `build-args: SERVICE=` +and the tag `service`. -To deploy a pull request as soon as it merges, add the label `deploy` to it -before the merge. After the merge, CI tests the commit and builds its image -as usual, then calls `deploy.yml` for that commit. A merge without the label -deploys nothing. In `ci.yml`: +### The Deploy workflow -```yaml - deploy: - needs: image - if: github.event_name == 'push' - permissions: - contents: read - id-token: write - pull-requests: read - uses: ./.github/workflows/deploy.yml - with: - commit: ${{ github.sha }} - pr-label: deploy - secrets: inherit -``` +`deploy.yml` starts in 2 ways: -For this, `deploy.yml` also has a `workflow_call` trigger with the inputs -`commit` and `pr-label`, and grants `pull-requests: read`. It passes both to -`deploy-ecs.yml`, as the template does. `deploy-ecs.yml` checks the label in -a job of its own, so a merge without the label shows no deployment in -GitHub. +| Start | Check | +| --- | --- | +| A person, from the Actions tab or `gh workflow run deploy.yml -f commit=` | None. An empty commit means the newest on the default branch. | +| The `deploy` job of `ci.yml`, after the `image` job of a merge | The merged pull request must have the label `deploy`. | + +Without the label, the deploy is skipped and GitHub shows no deployment. A +repo with no `deploy.yml` never deploys. The workflow stops if the commit is +not on the default branch, or if the image of the commit is not in ECR. It +never waits for an image: after a merge, the `deploy` job starts only when the +`image` job of the same run has pushed it. To roll back, deploy an older commit +by hand. + +The template is `workflow-templates/service-deploy.yml`. Inputs of +`deploy-ecs.yml`: `service`, `commit`, `cluster`, `ecs-service`, +`task-definition`, `container`, `container-env` (environment variables for +the container, one `KEY=VALUE` per line) and `task-definition-only` (registers +a task definition without a service deploy). The workflow takes no secrets. +Create the label `deploy` (Issues, Labels, New label) and the environment +`production` (Settings, Environments) in each service repo. + +A repo with several services calls `deploy-ecs.yml` once for each, with a +`strategy.matrix` or with separate jobs. A step that only one repo needs, such +as the atlas upload of static files and the Cloudflare purge, is a job in that +repo's `deploy.yml` before or after the call. ### A release -The template is `workflow-templates/release.yml`. It tests the tagged -commit, then calls `release.yml`. Write the changes under `## [1.2.3]` in -`CHANGELOG.md` before you push the tag `v1.2.3`. +Write the changes under `## [1.2.3]` in `CHANGELOG.md`, merge them, wait for +CI, then push the tag `v1.2.3`. The template is `workflow-templates/release.yml`. +The release fails unless the tagged commit has a passing `Checking the code`, so the repo +does not run its checks again. The caller grants `contents: write`, +`checks: read` and, for an image, `id-token: write`. | Repo | Inputs | | --- | --- | | go-commons | none: the notes only | -| nebula | `goreleaser: true`, `ecr-service: nebula` | -| eth-das-guardian | `goreleaser: true`, `dockerhub-repository: probelab/eth-das-guardian`, and the `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` secrets | - -### Terraform - -No repo runs Terraform in CI. probelab-infra runs Terraform from a local -shell. A repo that plans on every pull request, and applies when a person -starts the workflow on `main`, calls `terraform.yml` like this: - -```yaml -on: - pull_request: - paths: ["tf/**"] - workflow_dispatch: - -jobs: - plan: - if: github.event_name == 'pull_request' - permissions: - contents: read - id-token: write - pull-requests: write - uses: probe-lab/.github/.github/workflows/terraform.yml@v1 - with: - mode: plan - aws-role: arn:aws:iam:::role/ - secrets: inherit - - apply: - if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' - permissions: - contents: read - id-token: write - uses: probe-lab/.github/.github/workflows/terraform.yml@v1 - with: - mode: apply - aws-role: arn:aws:iam:::role/ - secrets: inherit -``` - -Every secret whose name starts with `TF_VAR_` becomes a Terraform variable. -The plan job runs the Terraform code of the pull request. Give it a read-only -role, and make sure that the trust policy of the apply role rejects pull -request runs. +| nebula | `ecr-service: nebula` | +| priv-hermes | `ecr-service: hermes`, `platforms: linux/amd64` | ## Runners -Checks and image builds run on the runner that the `runner` input names. The -templates pass `${{ vars.RUNNER_LABEL }}`. The org variable `RUNNER_LABEL` is +Checks and image builds run on the runner that the org variable `RUNNER_LABEL` +names, and on `ubuntu-latest` when the variable is empty. `RUNNER_LABEL` is `probelab-ci`, and only private repos can read it. A public repo gets an empty value and runs on GitHub-hosted runners, because public repos must not use the self-hosted runners. -Deploys, releases and Terraform always run on GitHub-hosted runners, because +Deploys and releases always run on GitHub-hosted runners, because they hold strong credentials. To move every check to GitHub-hosted runners, for example when the @@ -334,46 +140,36 @@ self-hosted machine is down, delete the org variable `RUNNER_LABEL`. ## Set up the org, once -1. Create the org variable `RUNNER_LABEL` with the value `probelab-ci` and +1. Merge this repo and tag the major version: `git tag v2.0.0 && git tag v2 + && git push origin v2.0.0 v2`. +2. Create the org variable `RUNNER_LABEL` with the value `probelab-ci` and the repository access "Private repositories". -2. In each service repo, create the environment `production` (Settings, - Environments). Deploys run in it, and GitHub shows the live commit under - Deployments on the repo page. -3. Narrow the AWS trust in probelab-infra (`tf/oidc.tf`), as the strategy's - first phase says. - -## Move a repo - -1. Replace the repo's workflows with `ci.yml` and, for a service, - `deploy.yml`. Delete the old files. For a service, also create the label - `deploy` (Issues, Labels, New label). -2. atlas and data-platform: remove the deploy on merge. Steps that only one - repo needs stay in that repo's own jobs, for example the atlas upload of - static files and the Cloudflare purge around its deploy. -3. Tests that use ClickHouse: read `CLICKHOUSE_PORT` and the other variables, - or map them in `test-command`, as nebula does above. The ports change on - every run. -4. Image tags become `-sha`. atlas (`atlas-`) and - data-platform (`dagster--`) change their format. The - first deploy after the move uses the new tag. -5. After the first green run, add `ci-ok` to the required checks of the org - ruleset. +3. In each service repo, create the environment `production` and the label + `deploy`. +4. Narrow the AWS trust in probelab-infra (`tf/oidc.tf`). +5. After the first green run of a repo, add `Checking the code` to the + required checks of the org ruleset. In a service repo, also add the image + check. Copy its name from a finished run. Remove `ci-ok` from the ruleset. ## Development -`just` lists the recipes. `just lint` runs actionlint on the workflows and -needs `actionlint` on the PATH. `just ci` runs the checks that can run -locally. The build, test and format recipes have nothing to do here: CI runs -the shared workflows against the projects in `tests/`. +`just` lists the recipes. `just lint` runs actionlint on the workflows and the +templates (`scripts/lint.sh`) and downloads `actionlint` if it is not on the PATH. `just ci` +runs the checks that can run locally. CI also builds a small image with +`image.yml`. ## Change this repo -1. Open a pull request. Its CI lints every workflow and template, and runs - `go.yml`, `python.yml` and `image.yml` against the projects in `tests/`. -2. `deploy-ecs.yml`, `release.yml` and `terraform.yml` need AWS, so the CI - only lints them. Before a release, try a change to them in one repo with - `@` instead of `@v1`. +1. Open a pull request. Its CI lints every workflow and template, and builds + the image in `tests/image/`. +2. `deploy-ecs.yml` and `release.yml` need AWS or a tag, so the CI only lints + them. Before a release, try a change in one repo with `@` + instead of `@v2`. 3. Write the changes under a new `## [X.Y.Z]` section in `CHANGELOG.md`. - After the merge, tag `vX.Y.Z` and move `v1` to it: `git tag -f v1 vX.Y.Z && git push -f origin vX.Y.Z v1`. + After the merge, tag `vX.Y.Z` and move the major tag to it: + `git tag -f v2 vX.Y.Z && git push -f origin vX.Y.Z v2`. 4. A change that breaks a caller, such as a renamed or removed input, is a new - major version, `v2`. + major version. +5. Before you add an input, ask if the repo could do it in its own `ci.yml` or + `justfile`. An input is only for a generic idea, such as a build argument or + an ECS container variable, never for the tooling of one repo. diff --git a/justfile b/justfile index a447b7f..6e0ae74 100644 --- a/justfile +++ b/justfile @@ -10,11 +10,11 @@ build: # Run the tests test: - @echo "nothing to test here: CI runs the shared workflows against tests/" + @echo "nothing to test here: CI builds the image in tests/image/" # Check the code without changing files lint: - actionlint .github/workflows/*.yml + ./scripts/lint.sh # Format the code in place format: diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100755 index 0000000..24ff385 --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Lints the shared workflows and the workflow templates with actionlint. +# Downloads actionlint into a cache directory when it is not on the PATH. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +actionlint_version=1.7.12 + +if ! command -v actionlint >/dev/null; then + bin="${XDG_CACHE_HOME:-$HOME/.cache}/actionlint/$actionlint_version" + if [[ ! -x "$bin/actionlint" ]]; then + echo "Downloading actionlint $actionlint_version to $bin" + mkdir -p "$bin" + bash <(curl -sSfL "https://raw.githubusercontent.com/rhysd/actionlint/v$actionlint_version/scripts/download-actionlint.bash") "$actionlint_version" "$bin" + fi + export PATH="$bin:$PATH" +fi + +actionlint -color .github/workflows/*.yml + +# Lints the templates as a repo holds them: in .github/workflows, with the +# Deploy template saved as deploy.yml, and calling the shared workflows of +# this commit, so a wrong input name fails here. +dir="$(mktemp -d)" +trap 'rm -rf "$dir"' EXIT +mkdir -p "$dir/.github/workflows" +cp .github/workflows/{image,deploy-ecs,release}.yml "$dir/.github/workflows/" +for f in workflow-templates/*.yml; do + sed 's#probe-lab/.github/.github/workflows/\(.*\)\.yml@v2#./.github/workflows/\1.yml#' "$f" \ + > "$dir/.github/workflows/template-$(basename "$f")" +done +cp "$dir/.github/workflows/template-service-deploy.yml" "$dir/.github/workflows/deploy.yml" +git -C "$dir" init -q +cd "$dir" +actionlint -color .github/workflows/template-*.yml .github/workflows/deploy.yml diff --git a/tests/go/Dockerfile b/tests/go/Dockerfile deleted file mode 100644 index b1689e6..0000000 --- a/tests/go/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:1.26 AS build -WORKDIR /src -COPY go.mod ./ -COPY *.go ./ -RUN CGO_ENABLED=0 go build -o /fixture . - -FROM scratch -COPY --from=build /fixture /fixture -ENTRYPOINT ["/fixture"] diff --git a/tests/go/go.mod b/tests/go/go.mod deleted file mode 100644 index 3451076..0000000 --- a/tests/go/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module fixture - -go 1.26 diff --git a/tests/go/go.sum b/tests/go/go.sum deleted file mode 100644 index e69de29..0000000 diff --git a/tests/go/justfile b/tests/go/justfile deleted file mode 100644 index da42ee4..0000000 --- a/tests/go/justfile +++ /dev/null @@ -1,8 +0,0 @@ -# checks formatting and runs go vet -lint: - test -z "$(gofmt -l .)" - go vet ./... - -# runs the tests -test: - go test ./... diff --git a/tests/go/main.go b/tests/go/main.go deleted file mode 100644 index 4aae288..0000000 --- a/tests/go/main.go +++ /dev/null @@ -1,13 +0,0 @@ -// Command fixture is a tiny program that the CI of this repo builds and tests -// with the shared workflows. -package main - -import "fmt" - -func main() { - fmt.Println(greeting("ProbeLab")) -} - -func greeting(name string) string { - return "Hello, " + name -} diff --git a/tests/go/main_test.go b/tests/go/main_test.go deleted file mode 100644 index ea256e8..0000000 --- a/tests/go/main_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "io" - "net" - "net/http" - "os" - "strings" - "testing" - "time" -) - -func TestGreeting(t *testing.T) { - if got := greeting("ProbeLab"); got != "Hello, ProbeLab" { - t.Fatalf("greeting = %q", got) - } -} - -// TestClickHouse checks that the workflow started ClickHouse and gave the -// tests its ports. It skips when CLICKHOUSE_HTTP_PORT is not set. -func TestClickHouse(t *testing.T) { - httpPort := os.Getenv("CLICKHOUSE_HTTP_PORT") - if httpPort == "" { - t.Skip("CLICKHOUSE_HTTP_PORT is not set") - } - host := os.Getenv("CLICKHOUSE_HOST") - - resp, err := http.Get("http://" + net.JoinHostPort(host, httpPort) + "/ping") - if err != nil { - t.Fatalf("ping ClickHouse: %v", err) - } - defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("read the ping answer: %v", err) - } - if strings.TrimSpace(string(body)) != "Ok." { - t.Fatalf("ping answered %q", body) - } - - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, os.Getenv("CLICKHOUSE_PORT")), 5*time.Second) - if err != nil { - t.Fatalf("connect to the native port: %v", err) - } - _ = conn.Close() -} diff --git a/tests/image/Dockerfile b/tests/image/Dockerfile new file mode 100644 index 0000000..e8aaf5e --- /dev/null +++ b/tests/image/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine:3.22 +RUN echo fixture > /fixture diff --git a/tests/python/.python-version b/tests/python/.python-version deleted file mode 100644 index 24ee5b1..0000000 --- a/tests/python/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 diff --git a/tests/python/fixture.py b/tests/python/fixture.py deleted file mode 100644 index 69ec026..0000000 --- a/tests/python/fixture.py +++ /dev/null @@ -1,5 +0,0 @@ -"""A tiny module that the CI of this repo tests with the shared workflows.""" - - -def greeting(name: str) -> str: - return f"Hello, {name}" diff --git a/tests/python/justfile b/tests/python/justfile deleted file mode 100644 index 0788117..0000000 --- a/tests/python/justfile +++ /dev/null @@ -1,7 +0,0 @@ -# checks that every file compiles -lint: - uv run python -m compileall -q fixture.py tests - -# runs the tests -test: - uv run python -m unittest discover -s tests -t . -v diff --git a/tests/python/pyproject.toml b/tests/python/pyproject.toml deleted file mode 100644 index e1046ad..0000000 --- a/tests/python/pyproject.toml +++ /dev/null @@ -1,6 +0,0 @@ -[project] -name = "fixture" -version = "0.0.0" -description = "A tiny project that the CI of this repo tests with the shared workflows." -requires-python = ">=3.12" -dependencies = [] diff --git a/tests/python/tests/__init__.py b/tests/python/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/python/tests/test_fixture.py b/tests/python/tests/test_fixture.py deleted file mode 100644 index 7691027..0000000 --- a/tests/python/tests/test_fixture.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import unittest -import urllib.request - -from fixture import greeting - - -class FixtureTest(unittest.TestCase): - def test_greeting(self): - self.assertEqual(greeting("ProbeLab"), "Hello, ProbeLab") - - @unittest.skipUnless(os.environ.get("CLICKHOUSE_HTTP_PORT"), "CLICKHOUSE_HTTP_PORT is not set") - def test_clickhouse(self): - """The workflow started ClickHouse and gave the tests its ports.""" - url = f"http://{os.environ['CLICKHOUSE_HOST']}:{os.environ['CLICKHOUSE_HTTP_PORT']}/ping" - with urllib.request.urlopen(url, timeout=10) as response: - self.assertEqual(response.read().decode().strip(), "Ok.") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/python/uv.lock b/tests/python/uv.lock deleted file mode 100644 index 82ab058..0000000 --- a/tests/python/uv.lock +++ /dev/null @@ -1,8 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.12" - -[[package]] -name = "fixture" -version = "0.0.0" -source = { virtual = "." } diff --git a/workflow-templates/go-library-ci.properties.json b/workflow-templates/go-library-ci.properties.json index b8f6520..e69de29 100644 --- a/workflow-templates/go-library-ci.properties.json +++ b/workflow-templates/go-library-ci.properties.json @@ -1,7 +0,0 @@ -{ - "name": "ProbeLab Go library CI", - "description": "Lints and tests every pull request and every merge of a Go library or tool.", - "iconName": "octicon checklist", - "categories": ["Go"], - "filePatterns": ["^go\\.mod$"] -} diff --git a/workflow-templates/go-library-ci.yml b/workflow-templates/go-library-ci.yml index ee4cf26..1c90f6e 100644 --- a/workflow-templates/go-library-ci.yml +++ b/workflow-templates/go-library-ci.yml @@ -1,4 +1,4 @@ -# CI for a Go library or tool that has no image: lints and tests every pull +# CI for a Go library or tool that has no image: runs `just ci` on every pull # request and every merge. name: CI @@ -17,22 +17,27 @@ permissions: contents: read jobs: - go: + # Runs the repo's own checks, the same ones a person runs with `just ci`. + # Extras that only this repo needs, such as a service container, go here. + check: + name: Checking the code if: github.event.pull_request.draft != true - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - # Empty in a public repo, which then uses GitHub-hosted runners. - runner: ${{ vars.RUNNER_LABEL }} - # Set it when `just lint` runs golangci-lint. - # golangci-lint: v2.15 - - # The one required check. It passes when every job passed or was skipped. - ci-ok: - name: ci-ok - if: always() - needs: [go] runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-latest' }} + timeout-minutes: 30 steps: - - name: Checking that every job passed - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 + - name: Checking out the code + uses: actions/checkout@v7 + + # A self-hosted runner keeps the Go caches on its disk, so only + # GitHub-hosted runners use the GitHub cache. + - name: Setting up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: ${{ runner.environment == 'github-hosted' }} + + - name: Setting up just + uses: extractions/setup-just@v4 + + - name: Running the checks + run: just ci diff --git a/workflow-templates/go-service-ci.properties.json b/workflow-templates/go-service-ci.properties.json index 2e69780..7143e3c 100644 --- a/workflow-templates/go-service-ci.properties.json +++ b/workflow-templates/go-service-ci.properties.json @@ -1,6 +1,6 @@ { "name": "ProbeLab Go service CI", - "description": "Lints and tests every pull request, and builds the image of every merge. Nothing deploys.", + "description": "Runs just ci on every pull request, and builds the image of every merge, and deploys a merge whose pull request has the label deploy.", "iconName": "octicon checklist", "categories": ["Go"], "filePatterns": ["^go\\.mod$", "^Dockerfile"] diff --git a/workflow-templates/go-service-ci.yml b/workflow-templates/go-service-ci.yml index 0dec063..a642d77 100644 --- a/workflow-templates/go-service-ci.yml +++ b/workflow-templates/go-service-ci.yml @@ -1,11 +1,10 @@ -# CI for a Go service: lints and tests every pull request, and builds the -# image of every merge. A merge deploys only when its pull request has the -# label "deploy"; otherwise a person deploys with the Deploy workflow. +# CI for a Go service: runs `just ci` on every pull request, and builds the +# image of every merge. A merge deploys when its pull request has the label "deploy": see deploy.yml. name: CI on: pull_request: - types: [opened, synchronize, reopened, ready_for_review] + types: [opened, synchronize, reopened, ready_for_review, labeled] push: branches: [$default-branch] @@ -18,29 +17,52 @@ permissions: contents: read jobs: - go: - if: github.event.pull_request.draft != true - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - # Empty in a public repo, which then uses GitHub-hosted runners. - runner: ${{ vars.RUNNER_LABEL }} - # Set it when `just lint` runs golangci-lint. - # golangci-lint: v2.15 + # Runs the repo's own checks, the same ones a person runs with `just ci`. + # Extras that only this repo needs, such as a service container, go here. + check: + name: Checking the code + # Runs unless the pull request is a draft, or the run comes from a label + # other than push-image. + if: >- + github.event.pull_request.draft != true && + (github.event.action != 'labeled' || github.event.label.name == 'push-image') + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checking out the code + uses: actions/checkout@v7 + + # A self-hosted runner keeps the Go caches on its disk, so only + # GitHub-hosted runners use the GitHub cache. + - name: Setting up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: ${{ runner.environment == 'github-hosted' }} + + - name: Setting up just + uses: extractions/setup-just@v4 + + - name: Running the checks + run: just ci image: - needs: go + name: Building the image + needs: check + # Runs when no check failed, and not for a draft pull request. + if: github.event.pull_request.draft != true && !failure() && !cancelled() permissions: contents: read id-token: write - uses: probe-lab/.github/.github/workflows/image.yml@v1 + uses: probe-lab/.github/.github/workflows/image.yml@v2 with: - runner: ${{ vars.RUNNER_LABEL }} service: ${{ github.event.repository.name }} - push: ${{ github.event_name == 'push' }} - # Deploys a merge when its pull request has the label "deploy". Without the - # label, a person deploys with the Deploy workflow. + # Deploys the merge after its image exists, when its pull request has the + # label "deploy". Delete this job in a repo that never deploys, and + # deploy.yml with it. deploy: + name: Deploying needs: image if: github.event_name == 'push' permissions: @@ -48,18 +70,4 @@ jobs: id-token: write pull-requests: read uses: ./.github/workflows/deploy.yml - with: - commit: ${{ github.sha }} - pr-label: deploy secrets: inherit - - # The one required check. It passes when every job passed or was skipped. - ci-ok: - name: ci-ok - if: always() - needs: [go, image] - runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-latest' }} - steps: - - name: Checking that every job passed - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 diff --git a/workflow-templates/python-service-ci.properties.json b/workflow-templates/python-service-ci.properties.json new file mode 100644 index 0000000..6ad5309 --- /dev/null +++ b/workflow-templates/python-service-ci.properties.json @@ -0,0 +1,7 @@ +{ + "name": "ProbeLab Python service CI", + "description": "Runs just ci on every pull request, and builds the image of every merge, and deploys a merge whose pull request has the label deploy.", + "iconName": "octicon checklist", + "categories": ["Python"], + "filePatterns": ["^pyproject\\.toml$", "^Dockerfile"] +} diff --git a/workflow-templates/python-service-ci.yml b/workflow-templates/python-service-ci.yml new file mode 100644 index 0000000..abb9853 --- /dev/null +++ b/workflow-templates/python-service-ci.yml @@ -0,0 +1,77 @@ +# CI for a Python service: runs `just ci` on every pull request, and builds +# the image of every merge. A merge deploys when its pull request has the label "deploy": see deploy.yml. +name: CI + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled] + push: + branches: [$default-branch] + +# A new push to a pull request cancels its running checks. +concurrency: + group: ci-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + # Runs the repo's own checks, the same ones a person runs with `just ci`. + # Extras that only this repo needs, such as a service container, go here. + check: + name: Checking the code + # Runs unless the pull request is a draft, or the run comes from a label + # other than push-image. + if: >- + github.event.pull_request.draft != true && + (github.event.action != 'labeled' || github.event.label.name == 'push-image') + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Checking out the code + uses: actions/checkout@v7 + + # A self-hosted runner keeps the uv cache on its disk, so only + # GitHub-hosted runners use the GitHub cache. + - name: Setting up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: ${{ runner.environment == 'github-hosted' }} + + - name: Setting up Python + uses: actions/setup-python@v7 + with: + python-version-file: .python-version + + - name: Setting up just + uses: extractions/setup-just@v4 + + - name: Running the checks + run: just ci + + image: + name: Building the image + needs: check + # Runs when no check failed, and not for a draft pull request. + if: github.event.pull_request.draft != true && !failure() && !cancelled() + permissions: + contents: read + id-token: write + uses: probe-lab/.github/.github/workflows/image.yml@v2 + with: + service: ${{ github.event.repository.name }} + + # Deploys the merge after its image exists, when its pull request has the + # label "deploy". Delete this job in a repo that never deploys, and + # deploy.yml with it. + deploy: + name: Deploying + needs: image + if: github.event_name == 'push' + permissions: + contents: read + id-token: write + pull-requests: read + uses: ./.github/workflows/deploy.yml + secrets: inherit diff --git a/workflow-templates/release.properties.json b/workflow-templates/release.properties.json index cd3fea8..6512573 100644 --- a/workflow-templates/release.properties.json +++ b/workflow-templates/release.properties.json @@ -1,6 +1,6 @@ { "name": "ProbeLab release", - "description": "Tests a version tag and publishes the GitHub release, with optional binaries and images.", + "description": "Publishes the GitHub release of a version tag whose commit passed CI, with binaries from goreleaser and an optional image.", "iconName": "octicon tag", "categories": ["Go"], "filePatterns": ["^CHANGELOG\\.md$"] diff --git a/workflow-templates/release.yml b/workflow-templates/release.yml index 86e9b07..5d9ba0e 100644 --- a/workflow-templates/release.yml +++ b/workflow-templates/release.yml @@ -1,7 +1,8 @@ # Releases a version tag. Write the changes under "## [1.2.3]" in -# CHANGELOG.md, then push the tag: `git tag v1.2.3 && git push origin v1.2.3`. -# It tests the tagged commit, then publishes the release. To retry a release, -# rerun the failed jobs of its run. +# CHANGELOG.md, merge them, wait for CI, then push the tag: +# `git tag v1.2.3 && git push origin v1.2.3`. The release fails unless CI +# passed for the tagged commit. To retry a release, rerun the failed jobs of +# its run. name: Release on: @@ -12,19 +13,13 @@ permissions: contents: read jobs: - go: - uses: probe-lab/.github/.github/workflows/go.yml@v1 - with: - runner: ${{ vars.RUNNER_LABEL }} - release: - needs: go + name: Releasing permissions: contents: write + checks: read id-token: write - uses: probe-lab/.github/.github/workflows/release.yml@v1 + uses: probe-lab/.github/.github/workflows/release.yml@v2 with: tag: ${{ github.ref_name }} - # goreleaser: true # ecr-service: SERVICE - secrets: inherit diff --git a/workflow-templates/service-deploy.yml b/workflow-templates/service-deploy.yml index 4954cb5..71f399d 100644 --- a/workflow-templates/service-deploy.yml +++ b/workflow-templates/service-deploy.yml @@ -1,7 +1,13 @@ -# The Deploy button: deploys the image of one commit to ECS. Start it from -# the Actions tab, or with `gh workflow run deploy.yml -f commit=`. -# ci.yml also calls it after a merge whose pull request has the label -# "deploy". Save it as .github/workflows/deploy.yml. +# The Deploy workflow: deploys the image of one commit to ECS. Save it as +# .github/workflows/deploy.yml. +# +# It starts in 2 ways: +# - A person starts it from the Actions tab, or with +# `gh workflow run deploy.yml -f commit=`. It deploys without a +# check. To roll back, deploy an older commit. +# - ci.yml calls it after the image job of a merge. It deploys only when the +# merged pull request has the label "deploy". +# A repo without this file never deploys. name: Deploy on: @@ -12,13 +18,6 @@ on: type: string required: false workflow_call: - inputs: - commit: - type: string - required: true - pr-label: - type: string - default: "" permissions: contents: read @@ -27,11 +26,11 @@ permissions: jobs: deploy: - uses: probe-lab/.github/.github/workflows/deploy-ecs.yml@v1 + name: Deploying + uses: probe-lab/.github/.github/workflows/deploy-ecs.yml@v2 with: service: ${{ github.event.repository.name }} commit: ${{ inputs.commit }} - pr-label: ${{ inputs.pr-label }} # Replace SERVICE with the names in probelab-infra. cluster: prod-use1-cmi-ecs-cluster ecs-service: prod-use1-cmi-SERVICE