Add workflow to trigger wheel releases

.github/workflows/release-dispatch.yml

@@ -0,0 +1,99 @@
+name: Dispatch Release
+
+# Reusable workflow — intended to be called by a release-trigger workflow in an
+# integration repository (integrations-core, integrations-extras, marketplace).
+
+on:
+  workflow_call:
+    inputs:
+      source-repo:
+        description: "Source repository name (e.g. integrations-core, integrations-extras, marketplace)"
+        required: false
+        type: string
+      packages:
+        description: >-
+          Packages to release. Accepts a JSON array (e.g. '["postgres","datadog_checks_base"]'),
+          'all' to release every Python package in the repo, or omit to auto-detect from new tags.
+        required: false
+        type: string
+      source-repo-ref:
+        description: "Commit SHA or ref to build from"
+        required: false
+        type: string
+      target:
+        description: "Target environment (dev = no tag push, dev S3; prod = push tags, prod S3)"
+        required: false
+        type: string
+        default: prod  # callers (e.g. release-trigger.yml) typically override this
+      dry-run:
+        description: >-
+          When true, print what would be released and where without pushing tags
+          or triggering downstream wheel builds.
+        required: false
+        type: boolean
+        default: false
+      ddev-version:
+        description: "ddev version, pinned by default."
+        required: false
+        type: string
+      is-stable-release:
+        description: "'true' for master/X.X.x (blocks pre-releases), 'false' for alpha/beta/rc (blocks stable). Unset defaults to stable behavior."
+        required: false
+        type: string
+
+permissions:
+  id-token: write
+  contents: write # ddev needs to push tags
+
+jobs:
+  dispatch:
+    name: Tag releases and dispatch wheel builds
+    runs-on: ubuntu-latest
+    environment: release
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0 # ddev needs full tag history
+
+      - name: Install ddev
+        run: pip install "ddev${{ inputs.ddev-version && format('=={0}', inputs.ddev-version) || '==14.3.2' }}"
+
+      - name: Configure ddev
+        env:
+          SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }}
+        run: |
+          REPO_SHORT="${SOURCE_REPO#integrations-}"
+          ddev config set upgrade_check false
+          ddev config set repos.${REPO_SHORT} .
+          ddev config set repo ${REPO_SHORT}
+
+      - name: Prepare dispatch
+        id: prepare
+        env:
+          TARGET: ${{ inputs.target }}
+          DRY_RUN: ${{ inputs.dry-run }}
+          SELECTED_PACKAGES: ${{ inputs.packages }}
+          SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }}
+          REF: ${{ inputs.source-repo-ref || github.sha }}
+          IS_STABLE_RELEASE: ${{ inputs.is-stable-release }}
+        run: python .github/workflows/scripts/release_prepare.py
+
+      - name: Get GitHub token via dd-octo-sts
+        if: steps.prepare.outputs.has_packages == 'true' && !inputs.dry-run
+        id: octo-sts
+        uses: DataDog/dd-octo-sts-action@08f2144903ced3254a3dafec2592563409ba2aa0 # v1.0.1
+        with:
+          scope: DataDog/agent-integration-wheels-release
+          policy: integrations-core.dispatch-wheel-builds
+
+      - name: Dispatch release
+        if: steps.prepare.outputs.has_packages == 'true'
+        env:
+          GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
+          PACKAGES: ${{ steps.prepare.outputs.packages }}
+          SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }}
+          REF: ${{ inputs.source-repo-ref || github.sha }}
+          TARGET: ${{ inputs.target }}
+          DRY_RUN: ${{ inputs.dry-run }}
+        run: python .github/workflows/scripts/release_dispatch.py

.github/workflows/release-trigger.yml

@@ -0,0 +1,79 @@
+name: Trigger Wheel Builds
+
+# Thin wrapper — all tagging and dispatch logic lives in release-dispatch.yml.
+# The pipeline can build wheels for any Python package in the repo.
+# Equivalent trigger workflows exist in integrations-extras and marketplace.
+
+on:
+  push:
+    branches:
+      - master
+      - "[0-9]+.[0-9]+.x"
+      - "alpha/*"
+      - "beta/*"
+      - "rc/*"
+    paths:
+      - "*/CHANGELOG.md"
+      - "*/datadog_checks/*/__about__.py"
+  workflow_dispatch:
+    inputs:
+      packages:
+        description: >-
+          Packages to release. JSON array (e.g. '["postgres","datadog_checks_base"]'),
+          'all' to release every Python package in the repo, or omit to auto-detect from new tags.
+        required: false
+        type: string
+      source-repo-ref:
+        description: "Commit SHA or ref to build from"
+        required: true
+        type: string
+      target:
+        description: "Target environment (dev or prod)"
+        required: false
+        type: choice
+        options:
+          - dev
+          - prod
+        default: dev
+      dry-run:
+        description: "Print what would be released without pushing tags or starting builds"
+        required: false
+        type: boolean
+        default: false
+      ddev-version:
+        description: "ddev version, pinned by default."
+        required: false
+        type: string
+
+jobs:
+  context:
+    name: Detect release context
+    runs-on: ubuntu-latest
+    outputs:
+      is-stable-release: ${{ steps.detect.outputs.is-stable-release }}
+    steps:
+      - id: detect
+        # Sets is-stable-release based on the branch: true for master/X.Y.x, false otherwise.
+        # Manual runs on master get "true", which blocks pre-release packages — conservative and intentional.
+        run: |
+          if [[ "$GITHUB_REF" =~ ^refs/heads/(master|[0-9]+\.[0-9]+\.x)$ ]]; then
+            echo "is-stable-release=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "is-stable-release=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  dispatch:
+    name: Release
+    needs: context
+    uses: ./.github/workflows/release-dispatch.yml
+    with:
+      source-repo: integrations-core
+      packages: ${{ inputs.packages || '' }}
+      source-repo-ref: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-ref || github.sha }}
+      target: ${{ inputs.target || 'dev' }}
+      dry-run: ${{ inputs.dry-run || false }}
+      ddev-version: ${{ inputs.ddev-version || '' }}
+      is-stable-release: ${{ needs.context.outputs.is-stable-release }}
+    permissions:
+      id-token: write
+      contents: write

.github/workflows/scripts/_release/__init__.py

@@ -0,0 +1 @@
+TARGET_REPO = "DataDog/agent-integration-wheels-release"

.github/workflows/scripts/_release/dispatch.py

@@ -0,0 +1,102 @@
+"""HTTP dispatch logic for repository_dispatch events."""
+import http.client
+import json
+import ssl
+import sys
+import time
+import urllib.error
+import urllib.request
+
+from . import TARGET_REPO
+
+BATCH_SIZE = 200
+DISPATCH_URL = f"https://api.github.com/repos/{TARGET_REPO}/dispatches"
+MAX_ATTEMPTS = 5
+
+
+class DispatchError(Exception):
+    """Raised when a dispatch request fails after all retry attempts."""
+
+
+def build_payload(batch: list[str], source_repo: str, ref: str, target: str) -> dict:
+    """Return the ``repository_dispatch`` payload for one batch."""
+    return {
+        "event_type": "build-wheels",
+        "client_payload": {
+            "packages": batch,
+            "source_repo": source_repo,
+            "source_repo_ref": ref,
+            "target": target,
+        },
+    }
+
+
+def _urlopen(req: urllib.request.Request) -> http.client.HTTPResponse:
+    """Thin urllib wrapper — exists so tests can patch it without touching stdlib."""
+    return urllib.request.urlopen(req)
+
+
+def send_dispatch(
+    payload: dict,
+    token: str,
+    *,
+    dispatch_url: str = DISPATCH_URL,
+    max_attempts: int = MAX_ATTEMPTS,
+) -> None:
+    """POST a single ``repository_dispatch`` event to the wheels-release repo.
+
+    Retries up to ``max_attempts`` times on 5xx errors with exponential backoff.
+    Raises ``DispatchError`` on 4xx errors or after exhausting retries.
+    """
+    req = urllib.request.Request(
+        dispatch_url,
+        data=json.dumps(payload).encode(),
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+        method="POST",
+    )
+    for attempt in range(1, max_attempts + 1):
+        try:
+            with _urlopen(req) as resp:
+                print(f"  Dispatched: HTTP {resp.status}")
+                return
+        except (urllib.error.HTTPError, urllib.error.URLError) as e:
+            if isinstance(e, urllib.error.URLError):
+                if isinstance(e.reason, ssl.SSLError):
+                    raise DispatchError(f"SSL error (non-retriable): {e}") from e
+                body = str(e)
+                code = 503
+            else:
+                body = e.read().decode()
+                code = e.code
+            if code < 500 or attempt == max_attempts:
+                print(f"HTTP {code}: {body}", file=sys.stderr)
+                raise DispatchError(f"HTTP {code}: {body}")
+            print(f"  HTTP {code} on attempt {attempt}/{max_attempts}, retrying...", file=sys.stderr)
+            time.sleep(2**attempt)
+
+
+def dispatch_in_batches(
+    packages: list[str],
+    source_repo: str,
+    ref: str,
+    target: str,
+    token: str,
+    batch_size: int = BATCH_SIZE,
+) -> None:
+    """Dispatch all packages to the wheels-release repo, batching if needed."""
+    if batch_size <= 0:
+        raise ValueError("batch_size must be > 0")
+    num_packages = len(packages)
+    if num_packages == 0:
+        return
+    total_batches = (num_packages + batch_size - 1) // batch_size
+    for batch_num, start in enumerate(range(0, num_packages, batch_size), 1):
+        end = min(start + batch_size, num_packages)
+        current_batch = packages[start:end]
+        print(f"\nBatch {batch_num}/{total_batches}:")
+        print("\n".join(f"  - {name}" for name in current_batch))
+        send_dispatch(build_payload(current_batch, source_repo, ref, target), token)

.github/workflows/scripts/_release/github.py

@@ -0,0 +1,36 @@
+"""GitHub Actions I/O helpers."""
+import os
+from pathlib import Path
+
+
+def set_outputs(**kwargs: str) -> None:
+    """Write key=value pairs to GITHUB_OUTPUT."""
+    path = os.environ.get("GITHUB_OUTPUT")
+    if path:
+        with Path(path).open("a") as f:
+            for key, value in kwargs.items():
+                f.write(f"{key}={value}\n")
+
+
+def write_summary(content: str) -> None:
+    """Append markdown content to GITHUB_STEP_SUMMARY."""
+    path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if path:
+        with Path(path).open("a") as f:
+            f.write(content + "\n")
+
+
+def parse_bool_env(name: str, default: bool = False) -> bool:
+    """Parse a boolean environment variable.
+
+    Accepts 'true'/'1'/'yes' as True and 'false'/'0'/'no' as False (case-insensitive).
+    Returns ``default`` when the variable is absent or empty.
+    """
+    val = os.environ.get(name, "").strip().lower()
+    if not val:
+        return default
+    if val in ("true", "1", "yes"):
+        return True
+    if val in ("false", "0", "no"):
+        return False
+    return default

.github/workflows/scripts/_release/packages.py

@@ -0,0 +1,69 @@
+"""Package detection and resolution logic."""
+import json
+import re
+import subprocess
+from pathlib import Path
+
+_VERSION_SUFFIX_RE = re.compile(r"-\d+\.\d+\.\d+.*$")
+
+
+def get_all_packages(root: Path = Path(".")) -> list[str]:
+    """Return sorted list of all Python packages found under *root*."""
+    return sorted(
+        {p.parent.parent.parent.name for p in root.glob("*/datadog_checks/*/__about__.py")}
+    )
+
+
+def get_tags_at_head() -> list[str]:
+    """Return the list of git tags pointing at HEAD."""
+    try:
+        return subprocess.check_output(["git", "tag", "--points-at", "HEAD"], text=True).splitlines()
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Failed to get git tags at HEAD: {e}") from e
+
+
+def detect_from_tags(tags: list[str]) -> list[str]:
+    """Return sorted unique package names extracted from release tags.
+
+    Tags are expected in the form ``<package>-X.Y.Z[...]``.
+    The version suffix is stripped to recover the package name.
+    """
+    return sorted({_VERSION_SUFFIX_RE.sub("", t) for t in tags if t.strip()})
+
+
+def resolve_packages(
+    selected: str,
+    all_packages: list[str],
+    head_tags: list[str] | None = None,
+) -> tuple[list[str], str]:
+    """Resolve the list of packages to release.
+
+    Resolution order:
+    - ``'all'`` / ``'ALL'`` → every package in the repo
+    - JSON array            → use the provided list verbatim
+    - empty string          → auto-detect from git tags at HEAD
+
+    Returns ``(packages, mode_description)``.
+    Raises ``ValueError`` on invalid input or unknown package names.
+    """
+    selected = selected.strip()
+
+    if selected.lower() == "all":
+        return all_packages, f"all ({len(all_packages)} packages in repo)"
+
+    if selected:
+        try:
+            packages = json.loads(selected)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"SELECTED_PACKAGES is not valid JSON: {e}") from e
+        mode = f"manual ({selected})"
+    else:
+        tags = head_tags if head_tags is not None else get_tags_at_head()
+        packages = detect_from_tags(tags)
+        mode = "auto-detect from tags at HEAD"
+
+    unknown = sorted(set(packages) - set(all_packages))
+    if unknown:
+        raise ValueError(f"Unknown packages: {', '.join(unknown)}")
+
+    return packages, mode