diff --git a/.github/workflows/hugo.yaml b/.github/workflows/hugo.yaml index 4f882230..c5d7ede9 100644 --- a/.github/workflows/hugo.yaml +++ b/.github/workflows/hugo.yaml @@ -1,7 +1,7 @@ -# Sample workflow for building and deploying a Hugo site to GitHub Pages name: Deploy Hugo site to Pages on: + pull_request: # Runs on pushes targeting the default branch push: branches: @@ -10,17 +10,14 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: contents: read - pages: write - id-token: write + pages: read -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +# PR checks have their own queue and cannot cancel a Pages publication. concurrency: - group: "pages" - cancel-in-progress: false + group: pages-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} # Default to bash defaults: @@ -30,14 +27,19 @@ defaults: jobs: # Build job build: + name: Build and check (${{ matrix.environment }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + environment: [production, preview] env: HUGO_VERSION: 0.164.0 steps: - name: Install Hugo CLI run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb + wget -O "$RUNNER_TEMP/hugo.deb" "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb" + sudo dpkg -i "$RUNNER_TEMP/hugo.deb" - name: Install Dart Sass run: sudo snap install dart-sass - name: Checkout @@ -45,9 +47,13 @@ jobs: with: submodules: recursive fetch-depth: 0 + persist-credentials: false - name: Setup Pages + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && matrix.environment == 'production' id: pages uses: actions/configure-pages@v6 + with: + enablement: false - name: Setup Node.js uses: actions/setup-node@v4 with: @@ -56,31 +62,41 @@ jobs: # so pin an LTS that supports it. node-version: "22" - name: Install Node.js dependencies - run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true" + run: npm ci - name: Download OpenAPI specs run: ./hack/download_openapi.sh env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build with Hugo + GITHUB_TOKEN: ${{ github.token }} + - name: Build and check documentation + # Fixtures select their own Hugo environment, independently of this step. env: - # For maximum backward compatibility with Hugo modules - HUGO_ENVIRONMENT: production - HUGO_ENV: production + HUGO_ENVIRONMENT: ${{ matrix.environment }} + HUGO_ENV: ${{ matrix.environment }} + PAGES_BASE_URL: ${{ steps.pages.outputs.base_url }} run: | - hugo \ - --gc \ - --minify \ - --baseURL "${{ steps.pages.outputs.base_url }}/" - - name: Mirror latest docs under /docs/ and /docs/v/ - # GitHub Pages cannot redirect, so the stable aliases are real copies. - run: ./hack/mirror_latest.sh public + base_url="${PAGES_BASE_URL:-https://cozystack.io}" + args=(--environment "$HUGO_ENVIRONMENT" --gc --minify) + if [ "$HUGO_ENVIRONMENT" = preview ]; then + base_url=https://preview.invalid + args+=(--buildFuture) + fi + base_url="${base_url%/}/" + hugo "${args[@]}" --baseURL "$base_url" + ./hack/mirror_latest.sh public + latest=$(hugo config --environment "$HUGO_ENVIRONMENT" --format json | jq -er '.params.latest_version_id') + python3 hack/test_docs_seo.py public --latest "$latest" \ + --environment "$HUGO_ENVIRONMENT" --base-url "$base_url" + - name: Check version rollover and legacy URL fixtures + if: matrix.environment == 'production' + run: python3 hack/test_docs_seo_fixtures.py - name: Patch robots.txt # if built in a contributor's repo and deployed to personal pages, disallow search indexing - if: github.repository_owner != 'cozystack' + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && matrix.environment == 'production' && github.repository_owner != 'cozystack' run: | echo -e "\nDisallow: /" >> public/robots.txt cat public/robots.txt - name: Upload artifact + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && matrix.environment == 'production' uses: actions/upload-pages-artifact@v5 with: path: ./public @@ -91,6 +107,10 @@ jobs: # Deployment job deploy: + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + permissions: + pages: write + id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63b5e065..dfb72e59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -204,7 +204,8 @@ npm install make serve # dev server at http://localhost:1313/docs # Production-shape build (reproduces what GitHub Pages serves) -./hack/download_openapi.sh && HUGO_ENV=production hugo --gc --minify +./hack/download_openapi.sh && hugo --environment production --gc --minify +./hack/mirror_latest.sh public # Pull upstream content (targets next/ by default; pass RELEASE_TAG to route) make update-all # trunk from upstream main @@ -289,16 +290,28 @@ stable canonical; absolute feed and print links follow the served copy. After a release, run Hugo and the mirror step together. Publishing just Hugo's output would leave the stable canonical and sitemap URLs without their pages. +The Pages workflow builds and checks both production and preview on every PR and before publishing from `main`. Preview includes `next/` and future content; it is never uploaded as the Pages deployment artifact. Both builds must pass before deployment. PR checks have read-only permissions and separate concurrency groups. + Check the production output and release rollover with the CI-pinned Hugo: ```sh -hugo --gc --minify --baseURL https://cozystack.io/ +hugo --environment production --gc --minify --baseURL https://cozystack.io/ ./hack/mirror_latest.sh public -python3 hack/test_docs_seo.py public --latest v1.6 # use latest_version_id +latest=$(hugo config --environment production --format json | jq -er '.params.latest_version_id') +python3 hack/test_docs_seo.py public --latest "$latest" --base-url https://cozystack.io/ HUGO=hugo python3 hack/test_docs_seo_fixtures.py ``` +Check preview output separately: + +```sh +hugo --environment preview --gc --minify --buildFuture --baseURL https://preview.invalid/ --destination public-preview +./hack/mirror_latest.sh public-preview +latest=$(hugo config --environment preview --format json | jq -er '.params.latest_version_id') +python3 hack/test_docs_seo.py public-preview --latest "$latest" --environment preview --base-url https://preview.invalid/ +``` + The fixture suite builds small sites to check minor-release rollover, translated copies, baseURL subpaths, previews, print output, HTTP responses without JavaScript, -and invalid configuration. The output check also scans the full site's sitemaps. +and invalid configuration. It also verifies that the output checker rejects missing legacy copies, wrong canonical URLs, indexable preview pages and noindex entries in sitemaps. The output check scans the full site's sitemaps and legacy URL registry, requires `next/` in previews and rejects it in production. Pass the actual publishing URL with `--base-url` when testing a custom domain or repository subpath. diff --git a/hack/test_docs_seo.py b/hack/test_docs_seo.py index 6994151d..9d74caee 100644 --- a/hack/test_docs_seo.py +++ b/hack/test_docs_seo.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Check the built site's docs canonical URLs, robots tags and sitemaps. +"""Check built docs URLs and production or preview indexing policy. Usage: python3 hack/test_docs_seo.py PUBLIC_DIR --latest v1.6 Run after Hugo and mirror_latest.sh. Uses only the Python standard library. @@ -7,10 +7,11 @@ import argparse from html.parser import HTMLParser +import json from pathlib import Path import re import unittest -from urllib.parse import unquote, urlsplit +from urllib.parse import unquote, urljoin, urlsplit import xml.etree.ElementTree as ET @@ -50,6 +51,8 @@ def handle_starttag(self, tag, attrs): class BuiltSiteTests(unittest.TestCase): public = None latest = None + environment = "production" + base_url = "https://cozystack.io/" def head(self, path): return Head((self.public / path).read_text()) @@ -65,8 +68,12 @@ def test_current_docs_canonical(self): for version in (self.latest, "", self.latest.split(".")[0]): with self.subTest(page=str(relative), version=version): page = self.head(Path("docs") / version / relative) + if self.environment == "preview": + self.assertEqual(page.canonical, []) + self.assertEqual(page.robots, ["noindex, nofollow"]) + continue self.assertEqual(len(page.canonical), 1) - self.assertEqual(urlsplit(page.canonical[0]).path, "/docs/" + str(relative).removesuffix("index.html")) + self.assertEqual(page.canonical, [urljoin(self.base_url, "docs/" + str(relative).removesuffix("index.html"))]) self.assertEqual(page.canonical, original.canonical) self.assertEqual(page.robots, ["index, follow"]) if version != self.latest: @@ -86,35 +93,82 @@ def test_archive_robots(self): continue count += 1 with self.subTest(page=str(source.relative_to(self.public))): - self.assertEqual(page.robots, ["noindex, follow"]) + self.assertEqual(page.robots, ["noindex, nofollow" if self.environment == "preview" else "noindex, follow"]) self.assertEqual(page.canonical, []) self.assertGreater(count, 0) def test_sitemap_matches_canonical(self): count = 0 latest_count = 0 + sitemap_count = 0 ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} for sitemap in self.public.rglob("sitemap.xml"): + sitemap_count += 1 root = ET.parse(sitemap).getroot() urls = [loc.text for loc in root.findall("s:url/s:loc", ns)] self.assertEqual(len(urls), len(set(urls))) + if self.environment == "preview": + self.assertEqual(urls, [], str(sitemap)) + continue for url in urls: count += 1 latest_count += "/docs/" in url and not re.search(r"/docs/(?:v[0-9]+|versions|next)(?:[./])", url) - path = unquote(urlsplit(url).path).lstrip("/") + self.assertTrue(url.startswith(self.base_url), url) + path = unquote(url.removeprefix(self.base_url)) with self.subTest(url=url): page = self.head(Path(path) / "index.html") self.assertEqual(page.canonical, [url]) self.assertEqual(page.robots, ["index, follow"]) + self.assertGreater(sitemap_count, 0) + if self.environment == "production": + self.assertGreater(count, 0) + self.assertGreater(latest_count, 0) + + def test_legacy_urls(self): + registry = Path(__file__).resolve().parents[1] / "data/docs-legacy-urls.json" + mapping = json.loads(registry.read_text()) + self.assertTrue(mapping) + for old, target in mapping.items(): + with self.subTest(url=old): + page = self.head(Path(old.lstrip("/")) / "index.html") + destination = self.head(Path(target.lstrip("/")) / "index.html") + self.assertFalse(page.refresh) + self.assertFalse(destination.refresh) + self.assertEqual(page.canonical, destination.canonical) + self.assertEqual(page.robots, destination.robots) + self.assertEqual(page.og_urls, destination.og_urls) + + def test_preview_docs_are_noindex(self): + if self.environment != "preview": + self.assertFalse((self.public / "docs/next").exists(), "next/ must not be published") + return + self.assertTrue((self.public / "docs/next/index.html").is_file(), "preview must build next/") + roots = [self.public / "docs"] + [ + path / "docs" for path in self.public.iterdir() + if path.is_dir() and (path / "docs").is_dir() + ] + count = 0 + for docs in roots: + for file in docs.rglob("*.html"): + page = Head(file.read_text()) + if page.refresh: + continue + count += 1 + with self.subTest(page=str(file.relative_to(self.public))): + self.assertEqual(page.robots, ["noindex, nofollow"]) + self.assertEqual(page.canonical, []) self.assertGreater(count, 0) - self.assertGreater(latest_count, 0) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("public", type=Path) parser.add_argument("--latest", required=True) + parser.add_argument("--environment", choices=("production", "preview"), default="production") + parser.add_argument("--base-url", default="https://cozystack.io/") args, remaining = parser.parse_known_args() BuiltSiteTests.public = args.public.resolve() BuiltSiteTests.latest = args.latest + BuiltSiteTests.environment = args.environment + BuiltSiteTests.base_url = args.base_url.rstrip("/") + "/" unittest.main(argv=[__file__, *remaining]) diff --git a/hack/test_docs_seo_fixtures.py b/hack/test_docs_seo_fixtures.py index 9b4c866b..993a0b86 100644 --- a/hack/test_docs_seo_fixtures.py +++ b/hack/test_docs_seo_fixtures.py @@ -8,6 +8,7 @@ from pathlib import Path import socket import subprocess +import sys import tempfile import threading import time @@ -529,6 +530,68 @@ def test_legacy_major_aliases_are_cleaned_on_major_rollover(self): self.assertIn("v2.0", (self.public / "docs/v2/reference/applications/mysql/index.html").read_text()) self.assertTrue((self.public / "docs/v1.6/applications/mariadb/index.html").is_file()) + def check_output(self, environment, success=True): + result = subprocess.run( + [sys.executable, str(REPO / "hack/test_docs_seo.py"), str(self.public), + "--latest", "v1.6", "--environment", environment, + "--base-url", "https://example.org/project/"], + capture_output=True, text=True) + if success: + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + else: + self.assertNotEqual(result.returncode, 0, "corrupted output passed checks") + return result.stdout + result.stderr + + def test_production_output_checker_rejects_corruption(self): + self.legacy_site() + self.configure(base="https://example.org/project/") + self.write("config/production/hugo.yaml", '''module: + mounts: + - source: content/en + target: content + lang: en + files: ['! docs/next/**'] + - source: content/ru + target: content + lang: ru + files: ['! docs/next/**'] +''') + self.build() + self.check_output("production") + guide = self.public / "docs/guide/index.html" + original = guide.read_text() + guide.write_text(original.replace("https://example.org/project/docs/guide/", "https://wrong.example/docs/guide/")) + self.assertIn("test_current_docs_canonical", self.check_output("production", success=False)) + guide.write_text(original) + old = self.public / "docs/reference/applications/mysql/index.html" + original = old.read_bytes() + old.unlink() + self.assertIn("test_legacy_urls", self.check_output("production", success=False)) + old.write_bytes(original) + sitemap = next(path for path in self.public.rglob("sitemap.xml") + if ET.parse(path).getroot().tag.endswith("urlset")) + original = sitemap.read_bytes() + tree = ET.parse(sitemap) + item = ET.SubElement(tree.getroot(), "{http://www.sitemaps.org/schemas/sitemap/0.9}url") + ET.SubElement(item, "{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text = "https://example.org/project/private/" + tree.write(sitemap) + self.assertIn("test_sitemap_matches_canonical", self.check_output("production", success=False)) + sitemap.write_bytes(original) + + def test_preview_output_checker_requires_next_and_noindex(self): + self.legacy_site() + self.configure(base="https://example.org/project/") + self.build("preview") + self.check_output("preview") + guide = self.public / "ru/docs/guide/index.html" + original = guide.read_text() + guide.write_text(original.replace("noindex, nofollow", "index, follow")) + self.assertIn("test_preview_docs_are_noindex", self.check_output("preview", success=False)) + guide.write_text(original) + next_page = self.public / "docs/next/index.html" + next_page.unlink() + self.assertIn("preview must build next/", self.check_output("preview", success=False)) + if __name__ == "__main__": unittest.main()