diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 46d9552..90c8a18 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -89,8 +89,19 @@ jobs: if: github.event_name != 'pull_request' uses: actions/configure-pages@v6 + # The docs live under a language prefix : English under /en/, built from source/, and every + # other language of docs_languages (source/_config.yml) under its own code, built from + # source/ with its translated pages laid over it (scripts/build-language.sh). English comes + # first : the other languages reuse its heading anchors. The install script, the Helm + # repository and the 404 page stay at the root, served from the English build. - name: Build with Jekyll - run: bundle exec jekyll build --source source --destination _site --baseurl "${{ steps.pages.outputs.base_path }}" + run: | + set -euo pipefail + scripts/build-language.sh en _site "${{ steps.pages.outputs.base_path }}" + for code in $(ruby -ryaml -e 'puts YAML.load_file("source/_config.yml")["docs_languages"].map { |l| l["code"] } - ["en"]'); do + scripts/build-language.sh "$code" _site "${{ steps.pages.outputs.base_path }}" + scripts/i18n-status.sh "$code" + done env: JEKYLL_ENV: production @@ -101,6 +112,13 @@ jobs: env: JEKYLL_ENV: production + # Every page that used to live at the root (ansibleforms.com/seed) keeps its address, + # as a page that sends the browser on to /en/. Released apps link to these addresses. + - name: Redirect the old addresses to /en/ + env: + SITE_URL: ${{ steps.pages.outputs.base_url || 'https://ansibleforms.com' }} + run: scripts/redirects.sh _site "${{ steps.pages.outputs.base_path }}" + # curl -fsSL https://ansibleforms.com/install | sh : source/install, served as it is. It # must reach the site unchanged (no front matter, so Jekyll copies it) and be valid sh. - name: The install script is served as it is diff --git a/.gitignore b/.gitignore index a5ead4f..058b7dc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ vendor/ _app/ _app6/ source/_data/help.yaml + +# A language other than English is assembled here before it is built (scripts/build-language.sh) +_build/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7233fc1..c202ebf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,6 +8,7 @@ lives in [ansibleforms/ansibleforms](https://github.com/ansibleforms/ansibleform - [What lives where](#what-lives-where) - [Branches and pull requests](#branches-and-pull-requests) - [Previewing the site](#previewing-the-site) +- [Translations](#translations) - [A change in the app that needs a page here](#a-change-in-the-app-that-needs-a-page-here) --- @@ -24,6 +25,13 @@ The site is everything under `source/`. Most of it is written here, but two file | `/helm-charts/` (the Helm chart repository) | the chart packages of the `ansibleforms/helm-charts` releases, indexed at build time | the `helm-charts` repository | | everything else in `source/` | this repository | here | +The docs are served under a language prefix (`/en/` for English), and the language selector in the header lists +every language in `docs_languages` (`source/_config.yml`). Every page that used to live at the root keeps +its address as a redirect, written at build time by `scripts/redirects.sh`, to the reader's language: the +one they last picked, else their browser's, else English. The install script (`/install`) and the Helm chart +repository (`/helm-charts/`) stay at the root, without a redirect. The translations are described in +[Translations](#translations). + The 6.x pages under `source/v6/` are a frozen copy of the maintenance line's docs. Only correct them when something there is plainly wrong for 6.x users. @@ -53,7 +61,33 @@ bundle install bundle exec jekyll serve --source source ``` -Then open . `source/_data/help.yaml` is ignored by git, so the copy never ends up in a commit. +Then open . The preview serves the English docs at the root, without the `/en/` prefix the published site adds. `source/_data/help.yaml` is ignored by git, so the copy never ends up in a commit. + +--- + +## Translations + +English is written in `source/`, and every other language lives next to it, built over a copy of the English pages. + +| File | What it holds | +|---|---| +| `i18n//` | the translated page, at the same path as its English page in `source/` | +| `i18n/.yml` | the language's settings: the menu group names, the site title, the version labels | +| `source/_data/ui.yml` | the site's own interface strings (table headings, previous and next, search) per language | +| `source/_includes/flag.html` | the flags, the same drawings as the app's language switcher | + +A page that is not translated yet shows in English, so the site never has a gap. Keep the translated page's +headings in the same order as the English ones: the build gives them their English anchors back by position +(`scripts/i18n-anchors.py`), so links and the language selector land on the same section in every language. + +Each translated page records which version of its English page it was translated from, in `translated_from:`. +When you change an English page, update its translations in the same pull request and stamp them again: + +```bash +scripts/i18n-status.sh es --stamp _configure/runners/registration.md +``` + +`scripts/i18n-status.sh es` lists every page not translated or out of date. The build prints the same list as warnings. --- diff --git a/README.md b/README.md index 7b951c8..7082ea2 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The site covers everything a user of AnsibleForms needs, from the first install - The app's changelog, rebuilt into the site on every release - The Helm chart repository, served under `/helm-charts/` - The 6.x documentation, kept alongside under `/v6/` +- The docs under a language prefix (`/en/`), the old addresses redirecting to the reader's language ## Previewing locally diff --git a/scripts/build-language.sh b/scripts/build-language.sh new file mode 100755 index 0000000..b447e8f --- /dev/null +++ b/scripts/build-language.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# ============================================================================= +# Build the docs of one language into // +# ============================================================================= +# +# English is written in source/ and built from there. Every other language is +# built from a copy of source/ with its translated pages laid over it +# (i18n//, the same paths as in source/) and its settings read after the +# English ones (i18n/.yml) : the layouts, includes, styles and images are +# shared, and a page that is not translated yet shows in English rather than +# going missing. +# +# The install script, the Helm repository page and the 404 page are served from +# the root of the site, never from a language : the English build hands them to +# the root, the other languages drop their copy. +# +# Spanish headings get their English anchors back afterwards (scripts/i18n-anchors.py), +# so a link to /es/page.html#some-section works like the English one. +# +# Usage: +# scripts/build-language.sh [] +# +# Args: +# code : the language, as listed in docs_languages (source/_config.yml) +# site directory : where the whole site is assembled, e.g. _site +# base path : the path the site is served under, empty for the custom domain +# +# Example: +# scripts/build-language.sh es _site "" +# ============================================================================= + +set -euo pipefail + +code="${1:?usage: scripts/build-language.sh []}" +site="${2:?usage: scripts/build-language.sh []}" +base="${3:-}" +out="$site/$code" + +# ----------------------------------------------------------------------------- +# English : straight from source/ +# ----------------------------------------------------------------------------- + +if [ "$code" = "en" ]; then + bundle exec jekyll build --source source --destination "$out" --baseurl "$base/en" + # Step 1 : the pages served from the root move there + mv "$out/install" "$out/helm-charts" "$out/404.html" "$site/" + exit 0 +fi + +# ----------------------------------------------------------------------------- +# Another language : source/ with i18n// laid over it +# ----------------------------------------------------------------------------- + +[ -f "i18n/$code.yml" ] || { echo "::error::i18n/$code.yml does not exist : no settings for language $code."; exit 1; } +[ -d "$site/en" ] || { echo "::error::$site/en does not exist : build English first, its anchors are reused."; exit 1; } + +# Step 1 : a copy of the English source, without the 6.x docs, with the translations on top +work="_build/$code" +rm -rf "$work" +mkdir -p "$work" +cp -a source/. "$work/" +rm -rf "$work/v6" "$work/.jekyll-cache" +if [ -d "i18n/$code" ]; then + cp -a "i18n/$code/." "$work/" +fi + +# Step 2 : the pages not translated yet stay under their translated parent in the menu +python3 scripts/i18n-fallback.py "$work" "$code" + +# Step 3 : the build, with the language's settings over the English ones +bundle exec jekyll build --source "$work" --config "source/_config.yml,i18n/$code.yml" \ + --destination "$out" --baseurl "$base/$code" + +# Step 4 : the pages served from the root are the English build's +rm -rf "$out/install" "$out/helm-charts" "$out/404.html" + +# Step 5 : the English anchors on the translated headings +python3 scripts/i18n-anchors.py "$site" "$code" diff --git a/scripts/i18n-anchors.py b/scripts/i18n-anchors.py new file mode 100755 index 0000000..d2b78a1 --- /dev/null +++ b/scripts/i18n-anchors.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# ============================================================================= +# Give the headings of a translated build their English anchors back +# ============================================================================= +# +# Kramdown builds a heading's anchor from its text, so a translated heading gets +# a translated anchor : "## Tokens de unión" becomes #tokens-de-unión. The +# translated pages keep their links in English (../runners/registration.html#join-tokens), +# and switching language should keep the reader on the same section, so every +# translated heading takes the anchor of the English heading it translates. +# +# A translated page has the same headings as its English page, in the same order, +# so the two are paired by position. A page whose heading count differs is left +# as it is and reported : it is out of step with its English page. +# +# Rewritten, in the built site of the language : +# - the id of each heading, and every same-page link to it (the table of +# contents, the anchor links next to the headings, links written in the page) ; +# - the section urls of the search index (assets/js/search-data.json). +# +# Usage: +# python3 scripts/i18n-anchors.py +# ============================================================================= + +import json +import re +import sys +from pathlib import Path + +# A heading with an id, as kramdown writes it :

(other attributes may come first) +HEADING_ID = re.compile(r']*?)\bid="([^"]*)"') + + +def heading_ids(html): + """Return the ids of the headings of a page, in document order. + + Args: + html: the page's HTML. + + Returns: + The list of heading ids. + """ + return [m.group(3) for m in HEADING_ID.finditer(html)] + + +def page_url(rel, code): + """Return the url a page is served at, as the search index writes it. + + Args: + rel: the page's path inside the language's build, e.g. runners/index.html. + code: the language code. + + Returns: + The url, e.g. /es/runners/. + """ + url = f"/{code}/{rel}" + return url[: -len("index.html")] if url.endswith("/index.html") else url + + +def main(): + """Rewrite the anchors of one language's build. + + Returns: + The process exit code. + """ + if len(sys.argv) != 3: + print("usage: python3 scripts/i18n-anchors.py ", file=sys.stderr) + return 2 + site, code = Path(sys.argv[1]), sys.argv[2] + en_root, lang_root = site / "en", site / code + + # ------------------------------------------------------------------------- + # Step 1 : the pages, heading by heading + # ------------------------------------------------------------------------- + by_url = {} # page url -> {translated anchor: English anchor} + pages = 0 + out_of_step = [] + for page in sorted(lang_root.rglob("*.html")): + rel = page.relative_to(lang_root).as_posix() + english = en_root / rel + if not english.is_file(): + continue + html = page.read_text(encoding="utf-8") + ours, theirs = heading_ids(html), heading_ids(english.read_text(encoding="utf-8")) + if len(ours) != len(theirs): + out_of_step.append(f"{rel} ({len(ours)} headings, English has {len(theirs)})") + continue + mapping = {o: t for o, t in zip(ours, theirs) if o != t} + if not mapping: + continue + + # one pass each, so an anchor renamed to another's old name is never renamed twice + html = HEADING_ID.sub( + lambda m: f' translated title. +# +# Usage: +# python3 scripts/i18n-fallback.py +# ============================================================================= + +import re +import sys +from pathlib import Path + +# One front matter line of the keys that matter here : title: Credentials, quoted or not +KEY_LINE = re.compile(r'^(title|parent|grand_parent):\s*(.*?)\s*$') + + +def front_matter_keys(text): + """Return the title, parent and grand_parent of a page's front matter. + + Args: + text: the page's content. + + Returns: + A dict of the keys found, their values unquoted. + """ + found = {} + if not text.startswith("---\n"): + return found + for line in text.split("\n")[1:]: + if line == "---": + break + m = KEY_LINE.match(line) + if m: + found[m.group(1)] = m.group(2).strip('"').strip("'") + return found + + +def main(): + """Rewrite the parents of the untranslated pages of one language. + + Returns: + The process exit code. + """ + if len(sys.argv) != 3: + print("usage: python3 scripts/i18n-fallback.py ", file=sys.stderr) + return 2 + work, code = Path(sys.argv[1]), sys.argv[2] + english_root, translated_root = Path("source"), Path("i18n") / code + + # Step 1 : English title -> translated title, from every translated page + titles = {} + for page in translated_root.rglob("*.md"): + rel = page.relative_to(translated_root) + english = english_root / rel + if not english.is_file(): + continue + en = front_matter_keys(english.read_text(encoding="utf-8")).get("title") + tr = front_matter_keys(page.read_text(encoding="utf-8")).get("title") + if en and tr: + titles[en] = tr + + # Step 2 : the parents of every page that is not translated + rewritten = 0 + for page in work.rglob("*.md"): + rel = page.relative_to(work) + if (translated_root / rel).is_file() or not (english_root / rel).is_file(): + continue + lines = page.read_text(encoding="utf-8").split("\n") + if not lines or lines[0] != "---": + continue + changed = False + for i, line in enumerate(lines[1:], start=1): + if line == "---": + break + m = KEY_LINE.match(line) + if m and m.group(1) in ("parent", "grand_parent"): + value = m.group(2).strip('"').strip("'") + if value in titles: + lines[i] = f'{m.group(1)}: "{titles[value]}"' + changed = True + if changed: + page.write_text("\n".join(lines), encoding="utf-8") + rewritten += 1 + + print(f"{code}: {rewritten} untranslated pages kept under their translated parent") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/i18n-status.sh b/scripts/i18n-status.sh new file mode 100755 index 0000000..3bde744 --- /dev/null +++ b/scripts/i18n-status.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# ============================================================================= +# Which translated pages are missing, or older than their English page +# ============================================================================= +# +# Every translated page (i18n//) carries, in its front matter, the git +# hash of the English page it was translated from : +# +# translated_from: 3f1c9e... +# +# When the English page changes, its hash no longer matches, and the translation +# is reported as stale until it is brought up to date and stamped again. A page +# not translated yet is reported as missing : the site shows it in English. +# +# Usage: +# scripts/i18n-status.sh report, as GitHub warnings +# scripts/i18n-status.sh --stamp record that (relative to +# source/) is up to date with its +# English page, after updating it +# +# Example: +# scripts/i18n-status.sh es --stamp _configure/runners/registration.md +# ============================================================================= + +set -euo pipefail + +code="${1:?usage: scripts/i18n-status.sh [--stamp ...]}" +shift + +# The English pages that are translated : everything but the 6.x docs, the pages served +# from the root only (404, Helm charts) and the changelog, which comes from the app +english_pages() { + (cd source && find . -name '*.md' \ + -not -path './v6/*' -not -path './.jekyll-cache/*' \ + -not -path './helm-charts/*' -not -name '404.md' -not -path './_reference/changelog.md' \ + | sed 's#^\./##' | sort) +} + +# ----------------------------------------------------------------------------- +# --stamp : write the English page's hash into the translation +# ----------------------------------------------------------------------------- + +if [ "${1:-}" = "--stamp" ]; then + shift + [ $# -gt 0 ] || { echo "usage: scripts/i18n-status.sh --stamp ..."; exit 2; } + for path in "$@"; do + page="i18n/$code/$path" + [ -f "$page" ] || { echo "$page does not exist"; exit 1; } + hash="$(git hash-object "source/$path")" + # Step 1 : drop the old stamp, then add the new one as the last front matter line + awk -v hash="$hash" ' + NR == 1 && $0 == "---" { fm = 1; print; next } + fm && /^translated_from:/ { next } + fm && $0 == "---" { print "translated_from: " hash; fm = 0 } + { print } + ' "$page" > "$page.tmp" && mv "$page.tmp" "$page" + done + exit 0 +fi + +# ----------------------------------------------------------------------------- +# The report +# ----------------------------------------------------------------------------- + +missing=0 +stale=0 +while IFS= read -r path; do + page="i18n/$code/$path" + if [ ! -f "$page" ]; then + echo "::warning file=source/$path::$code: not translated yet, shown in English" + missing=$((missing + 1)) + continue + fi + stamp="$(sed -n '2,/^---$/{s/^translated_from: *//p}' "$page")" + if [ "$stamp" != "$(git hash-object "source/$path")" ]; then + echo "::warning file=$page::$code: the English page changed since it was translated" + stale=$((stale + 1)) + fi +done < <(english_pages) + +echo "$code: $(english_pages | wc -l) pages, $missing not translated, $stale out of date" diff --git a/scripts/redirects.sh b/scripts/redirects.sh new file mode 100755 index 0000000..cdfca65 --- /dev/null +++ b/scripts/redirects.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# ============================================================================= +# Redirects from the old addresses to the docs in the reader's language +# ============================================================================= +# +# The docs used to live at the root of the site (ansibleforms.com/seed), and they +# now live under a language prefix (ansibleforms.com/en/seed, /es/seed). Released +# versions of the app, the installer, READMEs, search engines and bookmarks all +# still point at the old addresses, so every one of them has to keep working, and +# the root itself (ansibleforms.com/) is one of them. +# +# GitHub Pages cannot send real HTTP redirects, so this writes a small page at +# every old address instead. It sends the browser on at once, keeping the query +# string and the #anchor, to the same page in : +# 1. the language the reader last picked in the language selector, if any +# (assets/js/language-select.js keeps it in localStorage, af-lang) ; +# 2. otherwise the first of the browser's languages the docs are written in ; +# 3. otherwise English. +# Without JavaScript it goes to English. A canonical link points search engines at +# the English page, never at the redirect. +# +# A file that already exists at the root is never overwritten : /install, +# /helm-charts/ and /404.html stay where they are, served as they are. +# +# Usage: +# scripts/redirects.sh [] +# +# Args: +# site directory : the built site, holding the English docs in its en/ folder +# base path : the path the site is served under, empty for the custom domain +# +# Example: +# scripts/redirects.sh _site "" +# ============================================================================= + +set -euo pipefail + +site="${1:?usage: scripts/redirects.sh []}" +base="${2:-}" +url="${SITE_URL:-https://ansibleforms.com}" + +[ -d "$site/en" ] || { echo "::error::$site/en does not exist : build the English docs first."; exit 1; } + +# The languages built : every two-letter folder at the root of the site, as a JavaScript array +langs="$(find "$site" -mindepth 1 -maxdepth 1 -type d -regex '.*/[a-z][a-z]' -printf '"%f",' | sed 's/,$//')" + +# ----------------------------------------------------------------------------- +# One redirect page for every page +# ----------------------------------------------------------------------------- + +written=0 +kept=0 +while IFS= read -r -d '' page; do + # Step 1 : the old address of the page, relative to the site root + rel="${page#"$site/en/"}" + old="$site/$rel" + + # Step 2 : a file the site serves at the root itself wins over its redirect + if [ -e "$old" ]; then + kept=$((kept + 1)) + continue + fi + + # Step 3 : the page's address inside a language, with index.html dropped as the browser shows it + path="/$rel" + path="${path%index.html}" + target="$base/en$path" + + # Step 4 : the redirect page itself + mkdir -p "$(dirname "$old")" + cat > "$old" < + + + +Redirecting… + + + + + + +

This page has moved to $url/en$path.

+ + +EOF + written=$((written + 1)) +done < <(find "$site/en" -type f -name '*.html' -not -path "$site/en/404.html" -print0) + +echo "Wrote $written redirects to the languages [$langs], kept $kept pages already served at the root" diff --git a/source/_config.yml b/source/_config.yml index f1dce8a..754db59 100644 --- a/source/_config.yml +++ b/source/_config.yml @@ -45,11 +45,20 @@ docs_version: "7" docs_versions: - name: "7" title: "7.x (latest)" - url: "/" + url: "/en/" - name: "6" title: "6.x" url: "/v6/" +# Docs languages : the selector at the right of the header (_includes/language_select.html). +# Each language is its own build under // ; docs_language marks the one this site is +# written in. A language is listed here once its pages exist : its pages live in i18n//, +# its settings in i18n/.yml, and scripts/build-language.sh builds it. +docs_language: "en" +docs_languages: + - code: "en" + label: "English" + # Aux links (top right) aux_links: "GitHub": diff --git a/source/_data/ui.yml b/source/_data/ui.yml new file mode 100644 index 0000000..20e81fc --- /dev/null +++ b/source/_data/ui.yml @@ -0,0 +1,28 @@ +# The site's own interface strings, per docs language (docs_language in _config.yml) : the +# includes read them as site.data.ui[site.docs_language]. The page text itself is translated +# page by page, under i18n// ; a language added to docs_languages needs its block here. +en: + language: Language + version: Version + previous: "← Previous" + next: "Next →" + prev_next_label: Previous and next page + attribute: Attribute + comments: Comments + variable: Variable + choices_defaults: Choices/Defaults + choices: "Choices:" + default: "Default:" + default_choice: (default) + examples: "Examples:" + added: Added + secret: Secret + secret_title: "Holds a secret : keep it out of version control" + with_types: "Only available with types:" + search: Search + toc: Table of contents + on_this_page: On this page + no_results: No results found + theme: Theme + aux: {} + theme_credit: 'This site uses Just the Docs, a documentation theme for Jekyll.' diff --git a/source/_includes/af_prev_next.html b/source/_includes/af_prev_next.html index b1560f0..099ae46 100644 --- a/source/_includes/af_prev_next.html +++ b/source/_includes/af_prev_next.html @@ -32,18 +32,18 @@ {%- comment -%} Step 3 : the two cards ; Next stays in the right column when there is no Previous {%- endcomment -%} {%- if af_found -%} {%- if af_prev == "" -%}{%- assign af_prev = nil -%}{%- endif -%} -