diff --git a/.claude/rules/llm-docs-maintenance.md b/.claude/rules/llm-docs-maintenance.md index 5e5f533a6c8..1da5297eda5 100644 --- a/.claude/rules/llm-docs-maintenance.md +++ b/.claude/rules/llm-docs-maintenance.md @@ -63,3 +63,5 @@ So a relevant deep-dive actually gets read when a matching task comes up, each ` Granularity rule: the pointer inherits its host rule's `paths:`, so it must live in a rule scoped **no broader than the doc's topic** — otherwise it loads on unrelated work (a reveal.js pointer does not belong in a rule scoped to all of `src/format/**`). If the best existing rule is too generic, create a small dedicated rule scoped to that doc's source instead. **When adding a new `llm-docs/*.md`:** add its discovery pointer to a matching path-scoped rule, or create a narrowly-scoped rule for it. + +**When renaming an `##`/`###` heading:** discovery pointers may quote that heading text to route readers to a specific section (e.g. `built-version-ci.md` → `"Design decisions"`). Grep `.claude/rules/` for the old heading text and update any matching pointer — a stale quote fails loudly (grep/search finds nothing) but still needs fixing. diff --git a/.claude/rules/testing/built-version-ci.md b/.claude/rules/testing/built-version-ci.md new file mode 100644 index 00000000000..32341510986 --- /dev/null +++ b/.claude/rules/testing/built-version-ci.md @@ -0,0 +1,26 @@ +--- +paths: + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/test-ff-matrix.yml + - .github/workflows/create-release.yml + - .github/actions/build-dist-tarball/** + - .github/actions/merge-extension-tests/** +--- + +# Built-Version CI Workflows + +`llm-docs/built-version-testing-architecture.md` is the deep dive. Read only what the edit needs: + +- OS lists, bucket globs, concurrency-group suffixes, `runners:` inputs → "Built-mode test legs" (scheduler layout). +- Adding/reordering a leg, changing `quarto-install` resolution or artifact plumbing → "Flow diagrams" + "Built-mode test legs". +- Removing, inverting, or "simplifying" an existing mechanism → "Design decisions" (D1-D11) first — each records why the obvious alternative was rejected. + +Trivial one-line YAML edits: the invariants below plus the doc's "Document map" intro paragraph are enough. + +Invariants when editing: + +- Playwright and ff-matrix legs in `test-smokes-built.yml` must check both the source mode and `github.event.inputs.buckets == ''`. +- `test-ff-matrix.yml` owns the ff-matrix bucket glob. +- Scheduler jobs in `test-smokes-built.yml` set per-leg OS scope through their `runners:` inputs. +- Keep the per-call suffix in `test-ff-matrix.yml`'s concurrency group so sibling calls cannot cancel one another. diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index bec82f921b2..ffd4a4ec1b7 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -1,11 +1,12 @@ --- paths: - - "tests/**" + - tests/** --- # Test Infrastructure -Quarto's test suite lives in `tests/`. For comprehensive documentation, see `tests/README.md`. +Quarto's test suite lives in `tests/`. +For comprehensive documentation, see `tests/README.md`. ## Running Tests @@ -28,18 +29,23 @@ QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS $env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows ``` +**Binary mode:** set `QUARTO_TEST_BIN` to an installed Quarto outside the checkout. +With no test arguments, binary mode runs `smoke/`; pass Playwright or feature-format targets explicitly. +See `tests/README.md` and `llm-docs/built-version-testing-architecture.md`. + ## Test Types -| Type | Location | File Pattern | Details | -|------|----------|--------------|---------| -| Unit | `tests/unit/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | -| Smoke | `tests/smoke/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | -| Smoke-all | `tests/docs/smoke-all/` | `*.qmd` | `.claude/rules/testing/smoke-all-tests.md` | -| Playwright | `tests/integration/playwright/` | `*.spec.ts` | `.claude/rules/testing/playwright-tests.md` | +| Type | Location | File Pattern | Details | +| ---------- | ------------------------------- | ------------ | ------------------------------------------- | +| Unit | `tests/unit/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | +| Smoke | `tests/smoke/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | +| Smoke-all | `tests/docs/smoke-all/` | `*.qmd` | `.claude/rules/testing/smoke-all-tests.md` | +| Playwright | `tests/integration/playwright/` | `*.spec.ts` | `.claude/rules/testing/playwright-tests.md` | ## Dependencies -Tests require R, Python, and Julia. Run configuration script to set up: +Tests require R, Python, and Julia. +Run configuration script to set up: ```bash # Linux/macOS @@ -56,12 +62,13 @@ Managed via: ## Core Files -| File | Purpose | -|------|---------| -| `test.ts` | Test infrastructure (`testQuartoCmd`, `unitTest`) | -| `verify.ts` | Verification functions | -| `utils.ts` | Path utilities (`docs()`, `outputForInput()`) | -| `README.md` | Comprehensive documentation | +| File | Purpose | +| --------------- | ----------------------------------------------------------------------------------------- | +| `test.ts` | Test infrastructure (`testQuartoCmd`, `unitTest`) | +| `quarto-cmd.ts` | Quarto invocation dispatch (`runQuarto`; in-process dev vs `QUARTO_TEST_BIN` binary mode) | +| `verify.ts` | Verification functions | +| `utils.ts` | Path utilities (`docs()`, `outputForInput()`) | +| `README.md` | Comprehensive documentation | ## Debugging diff --git a/.claude/rules/testing/playwright-tests.md b/.claude/rules/testing/playwright-tests.md index 8cc3781a043..d74aab5e0c8 100644 --- a/.claude/rules/testing/playwright-tests.md +++ b/.claude/rules/testing/playwright-tests.md @@ -1,12 +1,13 @@ --- paths: - - "tests/integration/playwright/**/*.spec.ts" - - "tests/integration/playwright/**/*.ts" + - tests/integration/playwright/**/*.spec.ts + - tests/integration/playwright/**/*.ts --- # Playwright Tests -Browser-based tests for interactive features. Tests live in `tests/integration/playwright/tests/`. +Browser-based tests for interactive features. +Tests live in `tests/integration/playwright/tests/`. ## Local Development Workflow @@ -44,6 +45,10 @@ The wrapper (`playwright-tests.test.ts`): 3. Runs `npx playwright test` 4. Cleans up rendered output +The suite runs in dev and binary-mode CI. +Render spawns must use `quartoSpawnEnvOptions()` so a built Quarto cannot inherit dev-tree paths. +Browser assertions are ignored on Windows CI, so built-version CI has no Windows Playwright leg. + ## Test Structure Tests use `@playwright/test` framework: @@ -63,7 +68,8 @@ test("Feature description", async ({ page }) => { ### Parameterized Tests -When testing the same behavior across multiple formats or configurations, use `test.describe` with a test cases array instead of separate spec files. See `html-math-katex.spec.ts` and `axe-accessibility.spec.ts` for examples. +When testing the same behavior across multiple formats or configurations, use `test.describe` with a test cases array instead of separate spec files. +See `html-math-katex.spec.ts` and `axe-accessibility.spec.ts` for examples. ```typescript const testCases = [ @@ -81,11 +87,13 @@ test.describe('Feature across formats', () => { }); ``` -**When to use:** Same assertion logic applied to multiple formats, output modes, or configurations. Reduces file count and centralizes shared helpers. +**When to use:** Same assertion logic applied to multiple formats, output modes, or configurations. +Reduces file count and centralizes shared helpers. ### Expected Failures -Use `test.fail()` to document known failures. Playwright inverts the result: the test passes if it fails, and flags if it unexpectedly passes (signaling the fix landed). +Use `test.fail()` to document known failures. +Playwright inverts the result: the test passes if it fails, and flags if it unexpectedly passes (signaling the fix landed). ```typescript test('Feature that is known broken', async ({ page }) => { diff --git a/.claude/rules/testing/test-anti-patterns.md b/.claude/rules/testing/test-anti-patterns.md index e7523f9028e..152129beb4c 100644 --- a/.claude/rules/testing/test-anti-patterns.md +++ b/.claude/rules/testing/test-anti-patterns.md @@ -1,14 +1,25 @@ --- paths: - - "tests/**/*.ts" - - "tests/**/*.test.ts" + - tests/**/*.ts + - tests/**/*.test.ts --- # Test Anti-Patterns +## Don't: Rely on `execute()` Throwing to Fail a Test + +The harness converts errors from `TestDescriptor.execute` into log records. +Put assertions in `verify`, and include a log-reading verifier such as `noErrors` when using `runQuarto(..., { throwOnFailure: false })`. + +## Don't: Import `src/quarto.ts` in Tests + +Importing `src/quarto.ts` bypasses binary mode. +Use `testQuartoCmd()`/`runQuarto()` and resolve direct subprocess spawns with `quartoDevCmd()`. + ## Don't: Modify Environment Variables -`Deno.env.set()` modifies process-global state. Deno runs test files in parallel by default, so other tests can see modified values. +`Deno.env.set()` modifies process-global state. +Deno runs test files in parallel by default, so other tests can see modified values. **Details:** `llm-docs/testing-patterns.md` → "Environment Variable Testing Pitfalls" @@ -20,6 +31,9 @@ Never create `Project.toml`, `.venv/`, or `renv.lock` in test fixture directorie ## Don't: `Deno.chdir()` inside the test body -`Deno.chdir()` mutates process-global cwd, so a test that changes it can leak into other tests in the same process. The harness already changes and restores the working directory: return the directory from `TestContext.cwd`, create fixtures in `setup`, clean up in `teardown` (examples: `tests/unit/dotenv-config.test.ts`, `tests/smoke/use/template.test.ts`). For a temp directory you don't need to run *from*, use `withTempDir` (`tests/utils.ts`). A test that only needs a *relative* input can pass a path relative to the current cwd without changing it. +`Deno.chdir()` mutates process-global cwd, so a test that changes it can leak into other tests in the same process. +The harness already changes and restores the working directory: return the directory from `TestContext.cwd`, create fixtures in `setup`, clean up in `teardown` (examples: `tests/unit/dotenv-config.test.ts`, `tests/smoke/use/template.test.ts`). +For a temp directory you don't need to run *from*, use `withTempDir` (`tests/utils.ts`). +A test that only needs a *relative* input can pass a path relative to the current cwd without changing it. **Details:** `llm-docs/testing-patterns.md` → "Working-Directory-Sensitive Tests" diff --git a/.claude/rules/testing/typescript-tests.md b/.claude/rules/testing/typescript-tests.md index ea49883ba3d..6f32bee8dd4 100644 --- a/.claude/rules/testing/typescript-tests.md +++ b/.claude/rules/testing/typescript-tests.md @@ -1,12 +1,13 @@ --- paths: - - "tests/smoke/**/*.test.ts" - - "tests/unit/**/*.test.ts" + - tests/smoke/**/*.test.ts + - tests/unit/**/*.test.ts --- # TypeScript Tests -TypeScript-based tests using Deno. Smoke tests render documents; unit tests verify isolated functionality. +TypeScript-based tests using Deno. +Smoke tests render documents; unit tests verify isolated functionality. ## Running Tests @@ -22,17 +23,22 @@ TypeScript-based tests using Deno. Smoke tests render documents; unit tests veri ## Core Infrastructure -Core test files (`test.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. +Core test files (`test.ts`, `quarto-cmd.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. + +### Binary mode compatibility + +Smoke tests must work with either the dev sources or `QUARTO_TEST_BIN`: + +- Never `import { quarto } from "../../src/quarto.ts"` in tests — invoke quarto via `testQuartoCmd()`/`runQuarto()`. +- For subprocesses, use `quartoDevCmd()` or `quartoDevBinCmd()` and pass `quartoSpawnEnvOptions()`. +- `TestContext.requiresDevQuarto: true` ignores a test in binary mode — rare escape hatch for tests exercising quarto internals in-process. ### Search for an existing verifier before writing one -`verify.ts` already covers many output shapes — including parsed-content -verifiers, not just raw-text regex (e.g. `ensureIpynbCellMatches` JSON-parses a -notebook and matches against joined cell source; `ensureHtmlElements` / -`ensureHtmlSelectorSatisfies` parse the DOM). Before adding a new `Verify`, -grep `verify.ts` for the format or assertion you need. Reuse or extend the -existing helper rather than hand-rolling a near-duplicate. Same applies to -mock-context and fixture helpers in `tests/unit/**` and `tests/utils.ts`. +`verify.ts` already covers many output shapes — including parsed-content verifiers, not just raw-text regex (e.g. `ensureIpynbCellMatches` JSON-parses a notebook and matches against joined cell source; `ensureHtmlElements` / `ensureHtmlSelectorSatisfies` parse the DOM). +Before adding a new `Verify`, grep `verify.ts` for the format or assertion you need. +Reuse or extend the existing helper rather than hand-rolling a near-duplicate. +Same applies to mock-context and fixture helpers in `tests/unit/**` and `tests/utils.ts`. ## Smoke Tests (`tests/smoke/`) @@ -112,11 +118,14 @@ const markdownWithContent = asMappedString("# Title\nSome content"); **Mock Contexts:** -Several subsystems use context interfaces passed to functions. For unit tests, create `createMock*()` helpers with no-op stubs. Key pattern: async callbacks (like `withSpinner`) should just `await op()` so errors propagate normally. Check existing test files for helpers before writing new ones. - -| Context | Interface | Existing helpers | -|---------|-----------|-----------------| -| `ProjectContext` | `src/project/types.ts` | `tests/unit/project/utils.ts` → `createMockProjectContext()` | -| `InstallContext` | `src/tools/types.ts` | `tests/unit/tools/chrome-headless-shell.test.ts` → `createMockContext()` | -| `Format` | `src/config/types.ts` | `tests/unit/format-utils.ts` → `createMockFormat()` (wraps the real `createFormat()` from `src/format/formats-shared.ts` — no hand-rolled cast) | -| `TempContext` | `src/core/temp-types.ts` | No mock needed — use the real `createTempContext()` from `src/core/temp.ts` directly (see `tests/unit/sass-cache.test.ts`, `tests/unit/ral/safe-remove-dir.test.ts`) | +Several subsystems use context interfaces passed to functions. +For unit tests, create `createMock*()` helpers with no-op stubs. +Key pattern: async callbacks (like `withSpinner`) should just `await op()` so errors propagate normally. +Check existing test files for helpers before writing new ones. + +| Context | Interface | Existing helpers | +| ---------------- | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ProjectContext` | `src/project/types.ts` | `tests/unit/project/utils.ts` → `createMockProjectContext()` | +| `InstallContext` | `src/tools/types.ts` | `tests/unit/tools/chrome-headless-shell.test.ts` → `createMockContext()` | +| `Format` | `src/config/types.ts` | `tests/unit/format-utils.ts` → `createMockFormat()` (wraps the real `createFormat()` from `src/format/formats-shared.ts` — no hand-rolled cast) | +| `TempContext` | `src/core/temp-types.ts` | No mock needed — use the real `createTempContext()` from `src/core/temp.ts` directly (see `tests/unit/sass-cache.test.ts`, `tests/unit/ral/safe-remove-dir.test.ts`) | diff --git a/.github/actions/build-dist-tarball/action.yml b/.github/actions/build-dist-tarball/action.yml new file mode 100644 index 00000000000..7a5e1b6ca0c --- /dev/null +++ b/.github/actions/build-dist-tarball/action.yml @@ -0,0 +1,60 @@ +name: "Build Quarto dist tarball" +description: | + Builds a Quarto distribution tarball from the current checkout: + configure.sh, quarto-bld prepare-dist --set-version, then a tar.gz of + package/pkg-working (top-level directory quarto-) uploaded as a + workflow artifact. Shared by create-release.yml (make-tarball / + make-arm64-tarball) and test-smokes-built.yml. +inputs: + version: + description: "Version to stamp into the distribution (--set-version)" + required: true + arch: + description: "Target architecture: amd64 or arm64" + required: false + default: "amd64" + tarball-name: + description: "File name of the produced .tar.gz" + required: true + artifact-name: + description: "Workflow artifact name to upload the tarball as" + required: true + retention-days: + description: "Artifact retention in days (empty = repo default; upload-artifact ignores an empty value)" + required: false + default: "" +runs: + using: "composite" + steps: + - name: Configure + shell: bash + run: ./configure.sh + + - name: Prepare Distribution + shell: bash + run: | + pushd package/src/ + if [ "${{ inputs.arch }}" = "arm64" ]; then + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --arch aarch64 --log-level info + else + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --log-level info + fi + popd + + - name: Make Tarball + shell: bash + run: | + pushd package/ + mv pkg-working quarto-${{ inputs.version }} + tar --owner=root --group=root -czf "${{ inputs.tarball-name }}" quarto-${{ inputs.version }} + mv quarto-${{ inputs.version }} pkg-working + popd + + - name: Upload Artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact-name }} + path: ./package/${{ inputs.tarball-name }} + retention-days: ${{ inputs.retention-days }} + # the tarball is already gzipped - re-compressing wastes CPU + compression-level: 0 diff --git a/.github/actions/merge-extension-tests/action.yml b/.github/actions/merge-extension-tests/action.yml index 2091b9c95f8..fd2dbd0463b 100644 --- a/.github/actions/merge-extension-tests/action.yml +++ b/.github/actions/merge-extension-tests/action.yml @@ -11,6 +11,12 @@ runs: - name: Merge julia-engine tests shell: bash run: | + # The upstream tests do not sanitize the environment passed to quarto. + # Skip them until PumasAI/quarto-julia-engine strips the dev-tree vars. + if [[ -n "$QUARTO_TEST_BIN" ]]; then + echo "::notice title=julia-engine tests skipped in binary mode::subtree tests do not sanitize their quarto spawn env yet (see llm-docs/built-version-testing-architecture.md)" + exit 0 + fi SUBTREE=src/resources/extension-subtrees/julia-engine/tests cp -r "$SUBTREE/docs/julia-engine" tests/docs/julia-engine cp -r "$SUBTREE/smoke/julia-engine" tests/smoke/julia-engine diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 193d3e2ce67..66c30749b2b 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -24,13 +24,19 @@ on: - windows - macos default: all + smoke-artifacts-only: + description: "Build only the artifacts consumed by Smoke Tests (Built Version) nightly mode: the linux amd64 tarball and the signed Windows zip. Skips source/arm64 tarballs, deb/rpm installers, and the macOS build; the linux and Windows installer-verification jobs still run since they validate the artifacts this mode builds. For cheap signed builds of a branch (dispatch with publish-release=false); incompatible with publishing." + required: false + type: boolean + default: false env: NFPM_VERSION: "2.43.1" concurrency: # make publishing release concurrent (but others trigger not) - group: building-releases-${{ inputs.publish-release && 'prerelease' || github.run_id }} + # Partial test builds must not queue with releases. + group: building-releases-${{ inputs.publish-release && !inputs.smoke-artifacts-only && 'prerelease' || github.run_id }} jobs: configure: @@ -45,6 +51,13 @@ jobs: tag_pushed: ${{ steps.version_commit.outputs.tag_pushed }} if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'quarto-dev/quarto-cli') steps: + # Prevent a partial build from reaching the version commit and tag. + - name: Fail on publish-release + smoke-artifacts-only + if: ${{ inputs.publish-release && inputs.smoke-artifacts-only }} + run: | + echo "::error::smoke-artifacts-only builds cannot publish - re-dispatch with publish-release=false" + exit 1 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -116,6 +129,7 @@ jobs: default_author: github_actions make-source-tarball: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] steps: @@ -150,35 +164,18 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-amd64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-amd64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-amd64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: amd64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-amd64.tar.gz + artifact-name: Deb Zip make-arm64-tarball: runs-on: ubuntu-latest needs: [configure] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux') }} steps: - uses: actions/checkout@v6 with: @@ -188,30 +185,13 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --arch aarch64 --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-arm64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-arm64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Arm64 Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-arm64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: arm64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-arm64.tar.gz + artifact-name: Deb Arm64 Zip make-tarball-rhel: runs-on: ubuntu-latest @@ -266,7 +246,7 @@ jobs: make-installer-linux: runs-on: ubuntu-latest needs: [configure] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux') }} strategy: matrix: arch: [x86_64, aarch64] @@ -396,9 +376,7 @@ jobs: cargo build --release --all-features --manifest-path package/launcher/Cargo.toml cp package/launcher/target/release/quarto.exe package/pkg-working/bin/quarto.exe - # Scheduled runs never publish, so they skip DigiCert signing. Gated on - # `schedule` and not `publish-release`: those dispatches are the pre-merge - # check for bundled-binary bumps and must keep signing. + # Scheduled builds skip signing; dispatched no-publish builds keep it. - name: Sign files before making ZIP and MSI installer if: ${{ github.event_name != 'schedule' }} id: sign-files @@ -505,7 +483,7 @@ jobs: make-installer-mac: runs-on: macos-latest needs: [configure] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos') }} steps: - uses: actions/checkout@v6 with: @@ -575,7 +553,7 @@ jobs: test-zip-mac: runs-on: macos-latest needs: [configure, make-installer-mac] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos') }} steps: - uses: actions/checkout@v6 with: @@ -613,7 +591,8 @@ jobs: quarto --version publish-release: - if: ${{ inputs.publish-release }} + # never publish from a partial (smoke-artifacts-only) build + if: ${{ inputs.publish-release && !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [ configure, diff --git a/.github/workflows/test-ff-matrix.yml b/.github/workflows/test-ff-matrix.yml index a1195c4a59f..673e631dff1 100644 --- a/.github/workflows/test-ff-matrix.yml +++ b/.github/workflows/test-ff-matrix.yml @@ -1,5 +1,43 @@ name: Feature-Format Tests on: + # Reusable so built-version CI can invoke the same bucket definition. + workflow_call: + inputs: + extra-r-packages: + description: "extra R package to install for the runs (like a dev version of one of the deps) - comma separated, passed to renv::install" + required: false + type: string + default: "" + quarto-install: + description: "Which quarto to test (forwarded to test-smokes.yml): 'dev' (source tree, default), 'release', or 'artifact'" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is 'release' (forwarded to test-smokes.yml)" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Workflow artifact name containing the built quarto tarball/zip when quarto-install is 'artifact' (forwarded to test-smokes.yml)" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Workflow run id to download the artifact from (forwarded to test-smokes.yml; empty = the current run)" + required: false + type: string + default: "" + ref: + description: "Git ref to check out (forwarded to test-smokes.yml; empty means the default checkout behavior)" + required: false + type: string + default: "" + runners: + description: "JSON list of runner labels for the OS matrix (forwarded to test-smokes.yml)" + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: extra-r-packages: @@ -20,6 +58,7 @@ on: - ".github/workflows/performance-check.yml" - ".github/workflows/stale-needs-repro.yml" - ".github/workflows/test-bundle.yml" + - ".github/workflows/test-smokes-built.yml" - ".github/workflows/test-smokes-parallel.yml" - ".github/workflows/test-install.yml" - ".github/workflows/test-quarto-latexmk.yml" @@ -35,12 +74,15 @@ on: - ".github/workflows/stale-needs-repro.yml" - ".github/workflows/test-bundle.yml" - ".github/workflows/test-install.yml" + - ".github/workflows/test-smokes-built.yml" - ".github/workflows/test-smokes-parallel.yml" - ".github/workflows/test-quarto-latexmk.yml" - ".github/workflows/update-test-timing.yml" concurrency: - group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }} + # Reusable workflows evaluate concurrency in the caller's context, so each + # call needs a distinct suffix. + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}-${{ inputs.runners && format('call-{0}-{1}', github.run_id, inputs.runners) || 'dev' }} cancel-in-progress: true jobs: @@ -48,5 +90,13 @@ jobs: name: Run feature-format matrix uses: ./.github/workflows/test-smokes.yml with: + # Keep the feature-format glob owned by this workflow. buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]' extra-r-packages: ${{ inputs.extra-r-packages }} + # Non-call triggers have empty inputs and use dev defaults. + quarto-install: ${{ inputs.quarto-install || 'dev' }} + quarto-version: ${{ inputs.quarto-version }} + quarto-artifact-name: ${{ inputs.quarto-artifact-name }} + quarto-artifact-run-id: ${{ inputs.quarto-artifact-run-id }} + ref: ${{ inputs.ref }} + runners: ${{ inputs.runners || '["ubuntu-latest", "windows-latest"]' }} diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml new file mode 100644 index 00000000000..c365213d6d8 --- /dev/null +++ b/.github/workflows/test-smokes-built.yml @@ -0,0 +1,366 @@ +# Runs smoke, Playwright, and feature-format tests against a built Quarto. +# Sources: a build of the selected ref, artifacts from create-release, or a +# published release. See llm-docs/built-version-testing-architecture.md. +name: Smoke Tests (Built Version) +on: + # Test scheduled and manually dispatched create-release builds. + workflow_run: + workflows: ["Build Installers"] + types: [completed] + workflow_dispatch: + inputs: + source: + description: "Where the quarto under test comes from: build = build from this ref now; nightly = artifacts of a create-release run; release = published (pre-)release. The workflow_run trigger (after each nightly build) always uses nightly." + required: false + type: choice + options: + - build + - nightly + - release + default: build + buckets: + description: "Optional JSON list of test buckets/globs for a partial smoke run (e.g. [\"integration/playwright-tests.test.ts\"]); empty runs the full smoke suite plus the separate ff-matrix and Playwright legs" + required: false + type: string + default: "" + version: + description: "For source=release: 'pre-release' (latest prerelease), 'release' (latest stable), or a literal version like 1.9.10" + required: false + type: string + default: "pre-release" + run-id: + description: "For source=nightly: create-release run id to take artifacts from (empty = latest successful run)" + required: false + type: string + default: "" + +# required by the called test-smokes.yml (julia cache cleanup) +permissions: + actions: write + contents: read + +jobs: + # Build the selected ref (the dispatch default). + build-artifact: + name: Build quarto dist (linux-amd64) + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.rec.outputs.sha }} + steps: + - name: Checkout Repo + uses: actions/checkout@v6 + + - name: Record commit under test + id: rec + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Compute version marker + id: version + shell: bash + # Build metadata keeps quarto-required range checks valid. + run: echo "version=$(cat version.txt)+test.$(date +%Y%m%d)" >> "$GITHUB_OUTPUT" + + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball + with: + version: ${{ steps.version.outputs.version }} + arch: amd64 + tarball-name: built-quarto-linux-amd64.tar.gz + artifact-name: built-quarto-linux-amd64 + # Consumed only by this workflow run. + retention-days: 1 + + run-smokes-artifact: + name: Smoke tests against built artifact + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-playwright-artifact: + name: Playwright tests against built artifact + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + && github.event.inputs.buckets == '' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-artifact: + name: Feature-format matrix against built artifact + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + && github.event.inputs.buckets == '' + needs: [build-artifact] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + + # Test an existing published release. + resolve-release: + name: Resolve release version + if: github.event.inputs.source == 'release' + runs-on: ubuntu-latest + outputs: + version: ${{ steps.r.outputs.version }} + steps: + - name: Resolve version + id: r + shell: bash + env: + VERSION_INPUT: ${{ github.event.inputs.version }} + run: | + input="$VERSION_INPUT" + case "$input" in + pre-release | prerelease | "") + version="$(curl -fsSL https://quarto.org/docs/download/_prerelease.json | jq -r '.version')" + ;; + release) + version="$(curl -fsSL https://quarto.org/docs/download/_download.json | jq -r '.version')" + ;; + *) + version="$input" + ;; + esac + if [ -z "$version" ] || [ "$version" = "null" ]; then + echo "::error::could not resolve a quarto version from input '$input'" + exit 1 + fi + echo "Resolved version: $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Preflight - tag must have binary-mode harness support + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.r.outputs.version }} + # Older tags do not contain the binary-mode test harness. + run: | + if output="$(gh api "repos/${GITHUB_REPOSITORY}/contents/tests/quarto-cmd.ts?ref=refs/tags/v${VERSION}" --silent 2>&1)"; then + echo "refs/tags/v${VERSION} has binary-mode harness support" + elif echo "$output" | grep -q 'HTTP 404'; then + echo "::error::tests/quarto-cmd.ts not found at refs/tags/v${VERSION}: either the tag does not exist or the release predates binary-mode harness support - it cannot be tested in release mode" + exit 1 + else + echo "::error::gh api failed while checking refs/tags/v${VERSION}: $output" + exit 1 + fi + + run-smokes-release: + name: Smoke tests against published release + if: github.event.inputs.source == 'release' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest", "windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-playwright-release: + name: Playwright tests against published release + if: github.event.inputs.source == 'release' && github.event.inputs.buckets == '' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-release: + name: Feature-format matrix against published release + if: github.event.inputs.source == 'release' && github.event.inputs.buckets == '' + needs: [resolve-release] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest", "windows-latest"]' + + # Reuse artifacts from a completed create-release run. + resolve-nightly: + name: Resolve create-release run + # Only successful workflow_run builds are testable. + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') + runs-on: ubuntu-latest + outputs: + run-id: ${{ steps.r.outputs.run-id }} + sha: ${{ steps.r.outputs.sha }} + has-linux: ${{ steps.r.outputs.has-linux }} + has-windows: ${{ steps.r.outputs.has-windows }} + has-mac: ${{ steps.r.outputs.has-mac }} + steps: + - name: Resolve run id, commit and artifacts + id: r + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID_INPUT: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.event.inputs.run-id }} + run: | + run_id="$RUN_ID_INPUT" + if [ -z "$run_id" ]; then + run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow create-release.yml \ + --branch main --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')" + fi + if [ -z "$run_id" ]; then + echo "::error::no successful create-release run found to take artifacts from" + exit 1 + fi + sha="$(gh run view "$run_id" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha // empty')" + if [ -z "$sha" ]; then + echo "::error::could not resolve head_sha for run ${run_id}" + exit 1 + fi + # Run only legs whose artifacts still exist. + names="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts" \ + --paginate --jq '.artifacts[] | select(.expired | not) | .name')" + has() { grep -Fxq "$1" <<<"$names" && echo true || echo false; } + has_linux="$(has 'Deb Zip')" + has_windows="$(has 'Windows Zip')" + has_mac="$(has 'Mac Zip')" + if [ "$has_linux" = false ] && [ "$has_windows" = false ] && [ "$has_mac" = false ]; then + echo "::error::run ${run_id} produced none of the expected artifacts (Deb Zip / Windows Zip / Mac Zip)" + exit 1 + fi + echo "Using create-release run ${run_id} at commit ${sha} (linux=${has_linux} windows=${has_windows} mac=${has_mac})" + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "has-linux=$has_linux" >> "$GITHUB_OUTPUT" + echo "has-windows=$has_windows" >> "$GITHUB_OUTPUT" + echo "has-mac=$has_mac" >> "$GITHUB_OUTPUT" + + run-smokes-nightly-linux: + name: Smoke tests against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + # Publish runs add only a version/changelog commit after this SHA. + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-windows: + name: Smoke tests against nightly build (windows) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # Scheduled builds leave quarto.exe unsigned; dispatch builds sign it. + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-mac: + name: Smoke tests against nightly build (macOS) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # Signed and notarized; also provides the only macOS smoke coverage. + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + # Each caller job accepts one OS-specific artifact. + run-playwright-nightly-linux: + name: Playwright tests against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-playwright-nightly-mac: + name: Playwright tests against nightly build (macOS) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + # Playwright browser assertions are ignored on Windows CI. + + run-ff-matrix-nightly-linux: + name: Feature-format matrix against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + + run-ff-matrix-nightly-windows: + name: Feature-format matrix against nightly build (windows) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' + + # The feature-format toolchain is not validated on macOS. diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 5297f5f8f02..65fb10815f1 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -5,9 +5,10 @@ on: workflow_call: inputs: buckets: - description: "JSON string for buckets of tests to run in loop. Array of grouped tests." - required: true + description: "JSON string for buckets of tests to run in loop. Array of grouped tests. Empty means a full run." + required: false type: string + default: "" time-test: description: "Should we run tests to produce test file" required: false @@ -18,6 +19,36 @@ on: required: false type: string default: "" + quarto-install: + description: "Which quarto to test: 'dev' (source tree, default), 'release' (published release via quarto-actions/setup), or 'artifact' (built tarball from a workflow artifact)" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is 'release' (passed to quarto-dev/quarto-actions/setup)" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Workflow artifact name containing the built quarto tarball/zip when quarto-install is 'artifact'" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Workflow run id to download the artifact from (empty = the current run); lets a caller test artifacts built by another workflow run, e.g. a no-publish create-release build" + required: false + type: string + default: "" + ref: + description: "Git ref to check out (empty means the default checkout behavior)" + required: false + type: string + default: "" + runners: + description: "JSON runner matrix. macos-latest is reserved for scheduled built-version runs, not per-commit callers." + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: buckets: @@ -50,7 +81,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # Non-call triggers use the default runner matrix. + os: ${{ fromJSON(inputs.runners || '["ubuntu-latest", "windows-latest"]') }} time-test: - ${{ inputs.time-test }} exclude: @@ -63,6 +95,9 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v6 + with: + # empty (the default) keeps the standard checkout behavior + ref: ${{ inputs.ref }} - name: Fix temp dir to use runner one (windows) if: runner.os == 'Windows' @@ -85,16 +120,13 @@ jobs: with: node-version: 22 - # Playwright setup is only needed when this job can actually run the - # playwright suite: a bucket containing playwright-tests.test.ts, or a - # full run (empty buckets, which includes integration/). Windows never - # needs it: the suite is ignored on Windows CI (playwright-tests.test.ts, - # `ignore: gha.isGitHubActions() && isWindows`). + # Provision Playwright only for jobs that can run its assertions. - name: Cache multiplex server node_modules if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) uses: actions/cache@v5 with: path: tests/integration/playwright/multiplex-server/node_modules @@ -105,8 +137,9 @@ jobs: - name: Install node dependencies if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) run: npm install --loglevel=error --no-audit working-directory: ./tests/integration/playwright shell: bash @@ -114,8 +147,9 @@ jobs: - name: Get Playwright version if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) run: | VERSION=$(node -p "require('@playwright/test/package.json').version") echo "PLAYWRIGHT_VERSION=$VERSION" >> "$GITHUB_ENV" @@ -125,8 +159,9 @@ jobs: - name: Cache Playwright browsers if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) uses: actions/cache@v5 with: path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} @@ -137,8 +172,9 @@ jobs: - name: Install Playwright system dependencies if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) timeout-minutes: 15 run: npx playwright install-deps working-directory: ./tests/integration/playwright @@ -146,8 +182,9 @@ jobs: - name: Install Playwright Browsers if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) timeout-minutes: 15 run: npx playwright install working-directory: ./tests/integration/playwright @@ -196,6 +233,14 @@ jobs: sudo apt-get update -y sudo apt-get install -y librsvg2-bin + - name: Install missing system deps (macOS) + if: runner.os == 'macOS' + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + # PDF tests need poppler and librsvg beyond the preinstalled tools. + run: | + brew install poppler librsvg + - name: Restore R packages working-directory: tests run: | @@ -250,8 +295,122 @@ jobs: run: | uv sync --frozen + # The harness needs the dev runtime even when testing another binary. - uses: ./.github/workflows/actions/quarto-dev + - name: Set up release quarto + if: inputs.quarto-install == 'release' + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: ${{ inputs.quarto-version }} + + - name: Download built quarto artifact + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id == '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + + - name: Download built quarto artifact (from another run) + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id != '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + run-id: ${{ inputs.quarto-artifact-run-id }} + github-token: ${{ github.token }} + + - name: Install built quarto outside the checkout (Windows) + if: inputs.quarto-install == 'artifact' && runner.os == 'Windows' + shell: pwsh + # Git for Windows' bash ships GNU tar, which cannot read a zip archive, + # and the Windows artifact is always a zip - so this step runs under + # pwsh instead, where Expand-Archive handles it natively. + run: | + $zips = Get-ChildItem -Path "$env:RUNNER_TEMP/quarto-artifact" -Filter *.zip + if ($zips.Count -ne 1) { + Write-Host "::error::expected exactly one .zip in the quarto artifact, found: $($zips.Name -join ', ')" + exit 1 + } + $dest = "$env:RUNNER_TEMP/quarto-under-test" + Expand-Archive -Path $zips[0].FullName -DestinationPath $dest -Force + if (-not (Test-Path "$dest/bin")) { + $tops = Get-ChildItem -Path $dest -Force -Directory + if ($tops.Count -ne 1 -or -not (Test-Path (Join-Path $tops[0].FullName "bin"))) { + Write-Host "::error::could not locate bin/ in the extracted quarto artifact" + exit 1 + } + Get-ChildItem -Path $tops[0].FullName -Force | Move-Item -Destination $dest + } + Add-Content $env:GITHUB_PATH "$dest/bin" + + - name: Install built quarto outside the checkout + if: inputs.quarto-install == 'artifact' && runner.os != 'Windows' + shell: bash + # An in-checkout distribution would launch the dev sources. + run: | + mkdir -p "$RUNNER_TEMP/quarto-under-test" + shopt -s nullglob + artifacts=("$RUNNER_TEMP"/quarto-artifact/*.tar.gz "$RUNNER_TEMP"/quarto-artifact/*.zip) + if [ "${#artifacts[@]}" -ne 1 ]; then + echo "::error::expected exactly one .tar.gz or .zip in the quarto artifact, found: ${artifacts[*]:-none}" + exit 1 + fi + # Linux has a top directory; macOS archives are flat. + tar -xf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" + if [ ! -d "$RUNNER_TEMP/quarto-under-test/bin" ]; then + tops=("$RUNNER_TEMP"/quarto-under-test/*/) + if [ "${#tops[@]}" -ne 1 ]; then + echo "::error::expected exactly one directory in the extracted quarto artifact, found: ${#tops[@]}" + exit 1 + fi + top="${tops[0]%/}" + if [ ! -d "$top/bin" ]; then + echo "::error::could not locate bin/ in the extracted quarto artifact" + exit 1 + fi + shopt -s dotglob + mv "$top"/* "$RUNNER_TEMP/quarto-under-test"/ + fi + echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" + + - name: Pin and verify test target + if: inputs.quarto-install != 'dev' && inputs.quarto-install != '' + shell: bash + env: + QUARTO_VERSION_INPUT: ${{ inputs.quarto-version }} + run: | + v="$(quarto --version)" + echo "quarto under test: $v ($(command -v quarto))" + if [ "$v" = "99.9.9" ]; then + echo "::error::quarto on PATH reports the dev sentinel version 99.9.9 - the built/released quarto was not picked up (dev-mode trap)" + exit 1 + fi + # Only build metadata preserves quarto-required range semantics. + if ! echo "$v" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(\+[0-9A-Za-z.-]+)?$'; then + echo "::error::quarto version '$v' is not range-transparent semver (only build metadata '+...' is allowed; never a '-suffix' or a 4th component)" + exit 1 + fi + # Older tags do not contain the binary-mode harness. + if [ ! -f tests/quarto-cmd.ts ]; then + echo "::error::this checkout's test harness lacks binary-mode support (tests/quarto-cmd.ts is missing); release/artifact mode only works for refs that include the QUARTO_TEST_BIN harness support" + exit 1 + fi + # Convert the executable to a native Windows path for Deno.Command. + if [ "$RUNNER_OS" = "Windows" ]; then + bin="$(command -v quarto.exe || command -v quarto.cmd || command -v quarto)" + bin="$(cygpath -w "$bin")" + else + bin="$(command -v quarto)" + fi + echo "QUARTO_TEST_BIN=$bin" >> "$GITHUB_ENV" + # Verify every spawn when the caller requested a concrete version. + if [ -n "$QUARTO_VERSION_INPUT" ] \ + && [ "$QUARTO_VERSION_INPUT" != "pre-release" ] \ + && [ "$QUARTO_VERSION_INPUT" != "release" ]; then + echo "QUARTO_TEST_EXPECTED_VERSION=$QUARTO_VERSION_INPUT" >> "$GITHUB_ENV" + fi + - name: Install Tinytex env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -331,10 +490,16 @@ jobs: env: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUCKETS_JSON: ${{ inputs.buckets }} run: | haserror=0 failed_tests=() - readarray -t my_array < <(echo '${{ inputs.buckets }}' | jq -rc '.[]') + # while-read, not `readarray`: this step also runs on macOS, whose + # only bash is 3.2 (no mapfile/readarray builtin). + my_array=() + while IFS= read -r bucket; do + my_array+=("$bucket") + done < <(echo "$BUCKETS_JSON" | jq -rc '.[]') for file in "${my_array[@]}"; do echo "::group::Testing ${file}" echo ">>> ./run-tests.sh ${file}" @@ -369,10 +534,11 @@ jobs: env: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUCKETS_JSON: ${{ inputs.buckets }} run: | $haserror=$false $failed_tests=@() - foreach ($file in ('${{ inputs.buckets }}' | ConvertFrom-Json)) { + foreach ($file in ($env:BUCKETS_JSON | ConvertFrom-Json)) { Write-Host "::group::Testing ${file}" Write-Host ">>> ./run-tests.ps1 ${file}" ./run-tests.ps1 $file @@ -423,9 +589,9 @@ jobs: key: ${{ steps.cache-renv-packages-restore.outputs.cache-primary-key }} - uses: actions/upload-artifact@v7 - # Upload pLaywright test report if they exists (playwright is only running on Linux for now) + # Include the OS because sibling jobs share the workflow run. if: ${{ !cancelled() && runner.os != 'Windows' && hashFiles('tests/integration/playwright/playwright-report/**/*') != '' }} with: - name: playwright-report + name: playwright-report-${{ runner.os }} path: ./tests/integration/playwright/playwright-report/ retention-days: 30 diff --git a/dev-docs/checklist-make-a-new-quarto-prerelease.md b/dev-docs/checklist-make-a-new-quarto-prerelease.md index a7c0927cbe9..25c1398ad8f 100644 --- a/dev-docs/checklist-make-a-new-quarto-prerelease.md +++ b/dev-docs/checklist-make-a-new-quarto-prerelease.md @@ -13,5 +13,6 @@ - New release prerelease should be on Github at - A new tag should be on main for the new prerelease version - `version.txt` on main should have been updated by the workflow to the pre-release version just released: https://github.com/quarto-dev/quarto-cli/blob/main/version.txt +- (optional) validate the published prerelease: Actions -> "Smoke Tests (Built Version)" -> "Run Workflow" with source `release` (version defaults to `pre-release`) to run the smoke suite against the installed release binaries instead of the dev tree Note: Cloudsmith publishing is skipped for prereleases (only runs for stable releases). diff --git a/dev-docs/debugging-flaky-tests.md b/dev-docs/debugging-flaky-tests.md index 67e30dc33e4..2534b0f08bd 100644 --- a/dev-docs/debugging-flaky-tests.md +++ b/dev-docs/debugging-flaky-tests.md @@ -184,8 +184,17 @@ quarto install tinytex for test in test1.ts test2.ts test3.ts; do ./run-tests.sh $test || break done + +# Run against a built quarto instead of the in-process dev sources +# (binary mode; see llm-docs/built-version-testing-architecture.md) +QUARTO_TEST_BIN=/path/to/installed/quarto ./run-tests.sh path/to/test.ts ``` +By default the harness invokes quarto in-process; with `QUARTO_TEST_BIN` set, +each invocation is spawned as a subprocess of a built distribution. A flake +that disappears in binary mode points at in-process state pollution in the +dev harness rather than a product bug. + ### Package/Dependency Comparison ```bash diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md new file mode 100644 index 00000000000..ad6306ed35c --- /dev/null +++ b/llm-docs/built-version-testing-architecture.md @@ -0,0 +1,315 @@ +--- +main_commit: e5850df75 +analyzed_date: 2026-09-10 +key_files: + - tests/quarto-cmd.ts + - tests/test.ts + - tests/run-tests.sh + - tests/run-tests.ps1 + - tests/integration/playwright-tests.test.ts + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/test-ff-matrix.yml + - .github/workflows/create-release.yml + - .github/actions/build-dist-tarball/action.yml + - .github/actions/merge-extension-tests/action.yml +--- + +# Built-Version Testing Architecture + +This document explains how the test suite runs against a **built** Quarto distribution (binary mode) and records the design decisions behind the test harness and CI wiring. + +Document map: + +- **This doc** — architecture summary, flow diagrams, and design decisions. +- `tests/README.md` → "Binary mode" — local commands and authoring rules. +- `llm-docs/testing-patterns.md` → "Dev Mode vs Binary Mode" — authoring patterns for tests that must work in both modes. + +## Architecture in one paragraph + +Every `testQuartoCmd()`-based test invokes quarto through a single dispatch point, `runQuarto()` in `tests/quarto-cmd.ts`. +In **dev mode** (default) it calls the in-process `quarto()` entry point from `src/quarto.ts`. +In **binary mode** (`QUARTO_TEST_BIN` set to a built quarto extracted *outside* the checkout) it spawns that binary as a subprocess with `--log --log-format json-stream`, merges the child's log into the test's log file, and the verifiers run unchanged — they only ever see log records and rendered outputs. +CI-side, the reusable `test-smokes.yml` gained `quarto-install: dev | release | artifact` inputs (dev callers are untouched), and `test-smokes-built.yml` orchestrates three sources for the binary under test: `build` (build from this ref, dispatch only), `nightly` (reuse the packaged artifacts of a nightly `create-release` build — fires automatically via `workflow_run` after each one), and `release` (install a published (pre-)release at its tag) — fanning each source out to three test legs: smoke, playwright, and the feature-format matrix (see "Built-mode test legs"). + +## Flow diagrams + +### Test invocation + +Every `testQuartoCmd()`-based test goes through `runQuarto()`. +Verifiers read the same log records and rendered outputs in both modes. + +```mermaid +flowchart TB + subgraph deno ["Deno test process (tests/ harness, always runs from the repo checkout)"] + TQC["testQuartoCmd / testRender / testSite / smoke-all driver"] + RQ{"runQuarto()
tests/quarto-cmd.ts"} + LOG[("json-stream log file
{msg, level, levelName} per line")] + OUT[("rendered output files")] + VER["verifiers (tests/verify.ts)
noErrors, printsMessage, ensureHtmlElements, ..."] + end + DEV["in-process quarto()
imported from src/quarto.ts
dev TS sources, version 99.9.9"] + BIN["spawned subprocess: built quarto
--log file --log-format json-stream
dev env vars stripped (QUARTO_SHARE_PATH, DENO_DIR, ...)"] + + TQC --> RQ + RQ -->|"dev mode (default:
QUARTO_TEST_BIN unset)"| DEV + RQ -->|"binary mode
(QUARTO_TEST_BIN set)"| BIN + DEV --> LOG + DEV --> OUT + BIN --> LOG + BIN --> OUT + LOG --> VER + OUT --> VER +``` + +### Binary-mode test lifecycle + +```mermaid +sequenceDiagram + participant T as test() (tests/test.ts) + participant R as runQuarto() + participant Q as built quarto (subprocess) + participant V as verifiers + + T->>T: create temp json-stream log file + Note over T: harness logger disabled
(the child owns log capture) + T->>R: execute(logFile) + R->>Q: spawn QUARTO_TEST_BIN render ...
--log (per-invocation temp) --log-format json-stream
env = ambient minus dev-tree vars, plus TestContext.env + Q->>Q: render, write log records + output files + Q-->>R: exit (code, stdout/stderr drained) + R->>T: merge child log into the test log file + alt exit != 0 and no ERROR record in child log + R->>T: append synthetic ERROR record
(exit code + stderr tail) + end + alt timeout + R->>Q: kill process tree + R->>T: append timeout ERROR record + end + T->>V: verify(log records) + verify(output files) +``` + +### CI workflow + +```mermaid +flowchart LR + subgraph dev ["Dev mode: quarto = in-process TS sources"] + PR["PR / push"] --> TSP["test-smokes-parallel.yml
sharded buckets"] + DAILY["daily schedule"] --> TSfull["full run"] + end + subgraph built ["Binary mode: quarto = built distribution (QUARTO_TEST_BIN)"] + TSB["test-smokes-built.yml
after every nightly build + manual dispatch
smoke + playwright + ff-matrix"] + BUILDM["source: build
build linux-amd64 dist from this ref"] + NIGHTM["source: nightly
reuse packaged create-release artifacts"] + RELM["source: release
install published (pre-)release"] + TSB -->|"dispatch"| BUILDM + TSB -->|"after each nightly build
or dispatch"| NIGHTM + TSB -->|"dispatch"| RELM + end + CR["create-release.yml
scheduled build without publish
dispatch can publish or build test artifacts"] + ACT[".github/actions/build-dist-tarball
shared build recipe"] + + FFM["test-ff-matrix.yml (reusable)
owns the feature-format bucket"] + TS["test-smokes.yml (reusable)
inputs: install source, ref, runners,
buckets, artifact"] + TSP --> TS + DAILY --> TS + FFM --> TS + BUILDM -->|"smoke + playwright"| TS + NIGHTM -->|"smoke + playwright"| TS + RELM -->|"smoke + playwright"| TS + BUILDM -->|"ff-matrix"| FFM + NIGHTM -->|"ff-matrix"| FFM + RELM -->|"ff-matrix"| FFM + BUILDM -. uses .-> ACT + CR -. "tarball jobs use" .-> ACT + NIGHTM -. "downloads artifacts from" .-> CR +``` + +### `QUARTO_TEST_BIN` propagation in CI + +The "Pin and verify test target" step in `test-smokes.yml` computes `QUARTO_TEST_BIN` and exports it through `$GITHUB_ENV`. +Dev-mode callers skip the install and pin steps. + +```mermaid +flowchart TB + IN["workflow input
quarto-install: artifact | release"] + QD["quarto-dev action
provisions the harness Deno runtime"] + INST["install quarto under test
artifact: extract outside checkout
release: quarto-actions/setup"] + PIN["pin and verify test target
reject 99.9.9; check semver
export QUARTO_TEST_BIN"] + RTS["./run-tests.sh
binary-mode guard
default selection = smoke/"] + QC["tests/quarto-cmd.ts
runQuarto spawns QUARTO_TEST_BIN"] + + IN --> QD --> INST --> PIN --> RTS --> QC +``` + +## When to use which mode + +| Mode | Quarto under test | Trigger | Suites (legs) | Question answered | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | +| dev (`test-smokes.yml`) | in-process TS sources (99.9.9) | every PR/push + daily cron | everything (sharded per-commit; ff-matrix via its own cron/push/PR) | did this code change break behavior? | +| nightly | packaged nightly artifacts (Linux tarball, real `quarto.exe`, notarized Mac zip); Windows signing is skipped on the *scheduled* build, see D11 | automatic, after each nightly build | smoke (linux+windows+mac) + playwright (linux+mac) + ff-matrix (linux+windows) | does what we *ship* work? (bundling/packaging/launcher bugs; only macOS smoke coverage in CI) | +| build | fresh linux-amd64 dist from the current ref (unsigned) | manual dispatch | smoke + playwright + ff-matrix (all linux) | does this ref work when packaged? (works on forks/PR branches) | +| release | published (pre-)release via quarto-actions/setup, harness at its `v` tag | manual dispatch | smoke (linux+windows) + playwright (linux) + ff-matrix (linux+windows) | does the published version pass? | + +Dev mode and built modes are complementary, not redundant: dev uniquely covers `unit/`, `QUARTO_DEBUG` paths, the `quarto check` dev branch, and in-process races; built modes cover the packaged product dev mode never executes. +The playwright suite (`integration/playwright-tests.test.ts`) is no longer dev-only either: every built-mode source runs three suites ("legs") — smoke, playwright, and the feature-format matrix — see "Built-mode test legs" below. +The two other `tests/integration/` tests (`guess-chunk-options-format-document.test.ts`, `mermaid/github-issue-1340.test.ts`) still run only in the dev shards. + +In practice: + +- **Normally, no manual trigger is needed.** `nightly` runs after each nightly build and tests the packaged binaries on all three operating systems before publication. + Scheduled builds skip Windows signing (D11). +- **Dispatch `build`** when a branch touches packaging or the harness itself (`prepare-dist`, `configure`, `tests/quarto-cmd.ts`, ...) + and you want built-version feedback on *that ref* now. + Trades coverage (linux-only, unsigned) for immediacy and fork-friendliness. + To get *signed* Windows binaries for a branch instead, dispatch `create-release` with `publish-release=false` + `smoke-artifacts-only=true` — the `workflow_run` trigger then tests the build automatically (only the legs whose artifacts exist); a manual `source=nightly` dispatch with a `run-id` is just for re-testing an older run (see D7). +- **Dispatch `release`** after publishing for post-publish verification, e.g. the optional step in `dev-docs/checklist-make-a-new-quarto-prerelease.md`. + It works only for releases cut after the harness support merged (D10). + +## Built-mode test legs (scheduler layout) + +`test-smokes-built.yml` = the mode **resolvers** (build-artifact / resolve-nightly / resolve-release, unchanged) + a **scheduler**: per-leg caller jobs fanning out to the reusable workflows. +Three legs per source mode, each an independent job (one red suite never cancels the others): + +| leg | goes through | bucket | OS scope | +| ---------- | ------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| smoke | `test-smokes.yml` | `inputs.buckets` (empty = binary-mode `smoke/` default) | build: linux; nightly: linux+windows+mac (`has-*` gated); release: linux+windows | +| playwright | `test-smokes.yml` | `["integration/playwright-tests.test.ts"]` | linux (+ mac on nightly) — **never windows**, see below | +| ff-matrix | `test-ff-matrix.yml` (reusable) | owned by `test-ff-matrix.yml` | linux (+ windows on nightly/release) — no macOS (Julia/TeX toolchain unproven there) | + +Key points: + +- **The smoke leg doubles as the general bucket runner.** A manual dispatch + with the `buckets` input set runs only the smoke-slot jobs with that + bucket; the playwright + ff-matrix legs carry + `github.event.inputs.buckets == ''` in their `if:` and skip. +- **No windows playwright leg, deliberately.** The browser assertions are + hard-ignored on Windows CI (`playwright-tests.test.ts` + `ignore: gha.isGitHubActions() && isWindows`) — a windows leg would render + the corpus, skip every assertion, and report a misleading green. Rework + that gate (plus the `runner.os != 'Windows'` report-upload gate in + `test-smokes.yml`) before adding windows to the leg. +- **The playwright render wrapper needs the sanitized spawn env.** + `playwright-tests.test.ts` renders via `execProcess` + + `quartoDevCmd()` and MUST pass `quartoSpawnEnvOptions()`: without it the + built quarto inherits the dev-tree env (`QUARTO_SHARE_PATH`, ...) exported + by `run-tests.[sh|ps1]` for the harness and silently renders with + dev-tree resources (the D3/D4 dev-mode trap). +- **Playwright report artifacts are named per OS** + (`playwright-report-${{ runner.os }}`): several `test-smokes.yml` calls + share one workflow run in the fan-out, and duplicate artifact names make + `upload-artifact` fail even on green tests. +- **Per-leg OS scope is tuned in one place** — the `runners` inputs on the + scheduler jobs in `test-smokes-built.yml`. + +### `test-ff-matrix.yml` is reusable (`workflow_call`) + +The ff-matrix bucket glob (`../dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) is defined **only** in `test-ff-matrix.yml`; built-mode callers reuse it through its `workflow_call` trigger, which coexists with the dev triggers (cron/push/PR/dispatch). +Inputs `quarto-install`, `quarto-version`, `quarto-artifact-name`, `quarto-artifact-run-id`, `ref`, `runners`, `extra-r-packages` are forwarded verbatim to `test-smokes.yml`; the job uses `${{ inputs.x || }}` fallbacks so non-call triggers use the dev defaults. +Nesting depth `test-smokes-built.yml → test-ff-matrix.yml → test-smokes.yml` is 3, well within GitHub's reusable-workflow nesting limit. +A called workflow's top-level `concurrency` evaluates in the caller's context (`github.workflow`/`ref`/`run_id` are the caller run's), so the group carries a per-call suffix derived from `inputs.runners` + `github.run_id` — without it, every ff-matrix leg of one `test-smokes-built.yml` run would share a single cancel-in-progress group and could cancel a sibling leg. +Dev triggers get a constant `-dev` suffix (dedup semantics unchanged). +`test-ff-matrix.yml` declares no `permissions`, so the caller's `actions: write` (julia cache cleanup) flows through. + +## Design decisions + +Each entry: what was decided, why, and what would justify revisiting. + +### D1. Nightly wiring: `workflow_run`, not `workflow_call`/dispatch/inversion + +**Decision.** `test-smokes-built.yml` listens for completed "Build Installers" (`create-release.yml`) runs via `workflow_run` and reuses their artifacts cross-run (`quarto-artifact-run-id`). +The release pipeline is not modified for testing purposes. + +**Alternatives considered (2026-07, maintainer question):** + +- *create-release dispatches the test workflow at the end* — `workflow_run` hand-rolled: needs `actions: write` + `gh workflow run` code inside the release workflow, same default-branch constraint. + Strictly dominated. +- *create-release `workflow_call`s `test-smokes.yml` after building* — same-run artifacts (no run-id resolution) and an explicit DAG, but smoke results would redden `Build Installers` runs including real publishes; gating to schedule-only moves testing configuration (buckets, runner policy) into the release workflow permanently. +- *Inversion: `test-smokes-built` owns the daily schedule and calls `create-release` via `workflow_call`* — feasible (create-release uses the `inputs.` context exclusively, which works under `workflow_call`; its `github.event_name == 'schedule'` guard still behaves because a called workflow sees the caller's event). + Rejected because: (a) `smoke-artifacts-only` skips `make-installer-mac`, so the daily run would need the full build anyway — zero compute saved (there is no double build today: one nightly build, one test pass reusing its artifacts); + (b) it reverses the dependency — the nightly build is also a + release-pipeline canary (signing certs, notarization, installer tooling) and must not die when the test workflow is broken or paused; (c) nightly builds would disappear from the "Build Installers" run history. + +**Why `workflow_run` wins:** zero risk to the most sensitive workflow in the repo, and clean failure attribution — "Build Installers" red = the pipeline broke; "Smoke Tests (Built Version)" red = the product broke. + +**Known weaknesses (accepted):** the trigger couples on the workflow *display name* string (`workflows: ["Build Installers"]`; renaming create-release's `name:` silently stops the trigger), and "trigger never fired" is silent (mitigated by daily cadence — an absent run is visible). +Note the trigger fires on EVERY completed create-release run, not only nightly schedules — manual dispatches (including partial `smoke-artifacts-only` builds) get tested too, which is why each nightly OS leg is gated on its artifact actually existing in the resolved run. + +**Revisit when:** the system has a green track record and maintainers want one atomic nightly build-and-test signal — then the inversion is the principled consolidation, done as a deliberate follow-up. + +### D2. Version marker: semver *build metadata* (`X.Y.Z+test.YYYYMMDD`) + +Built test dists are stamped `$(cat version.txt)+test.$(date +%Y%m%d)`. +Never a `-suffix` (prerelease versions fail every plain `>=X.Y` `quarto-required` range — the vendored semver has no `includePrerelease` anywhere) and never a 4th dot component (not semver; the vendored `deno.land/x/semver@1.4.0` throws). +Build metadata is range-transparent for every gate while still distinguishable from the `99.9.9` dev sentinel. + +### D3. Dist outside the checkout + `99.9.9` sentinel refusal + +Installed launchers detect dev mode via a sibling `src/quarto.ts`: an in-repo `package/dist/bin/quarto` silently runs the TS sources instead of the built code. +Therefore the dist under test must be extracted *outside* the repo, and both `run-tests.[sh|ps1]` and `assertTestBinary()` refuse a binary reporting `99.9.9` (`kLocalDevelopment`). +CI extracts artifacts to `RUNNER_TEMP`. + +### D4. Child env: inherit ambient + strip dev vars (not clearEnv+allowlist) + +Binary-mode spawns inherit the ambient environment minus a strip list (`QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `DENO_DIR`, `QUARTO_DEBUG`, `QUARTO_FORCE_VERSION`, ...), with `TestContext.env` overlaid last. +A clearEnv+allowlist was rejected: the Windows system-variable surface (`SystemRoot`, `PATHEXT`, ...) +is unenumerable in practice. +The dev-tree exports in `run-tests.[sh|ps1]` are kept in all modes — the *harness* process still needs them; only the *child* is sanitized. + +### D5. Silent-green guard: synthetic ERROR records + +"Non-zero exit ⇒ ERROR record in the log" is NOT an invariant (pre-logger-init failures, `quarto add/remove` commandFailed, pandoc/typst passthroughs), and ~23% of smoke-all docs are verified only via the log. +So `runQuarto()` appends a synthetic `{level: 40}` record (exit code + stderr tail) when a non-zero exit leaves the child log record-free, and a timeout record when the process-tree kill fires (`pgrep -P` walk on POSIX — portable to macOS, unlike `ps --ppid`; `taskkill /T` on Windows). + +### D6. `QUARTO_TEST_BIN` is set at runtime, never declared statically + +The "Pin and verify test target" step in `test-smokes.yml` resolves the installed binary, verifies it (sentinel refusal, semver shape, optional `QUARTO_TEST_EXPECTED_VERSION` match), and exports it via `$GITHUB_ENV` so every later step — including the unchanged `run-tests.sh` invocation — sees it. +With `quarto-install: dev` (all pre-existing callers) these steps are skipped and nothing changes. + +### D7. `smoke-artifacts-only` is for cheap *branch* builds, not the daily path + +The `create-release.yml` input skips source/arm64 tarballs and the Mac installer, yielding a fast signed Linux+Windows build for on-demand testing of a branch: dispatch create-release with `publish-release=false` + `smoke-artifacts-only=true` and the `workflow_run` trigger tests the build automatically (mac leg skipped via the artifact-existence gate — one dispatch total). +Guards: `configure` fails fast if `publish-release` (which defaults to true) is combined with `smoke-artifacts-only` — otherwise the version commit+tag step would push an orphan tag — and such runs use a per-run concurrency group so they never queue in the shared `prerelease` group against a real release. +It deliberately does NOT feed the daily path: the daily needs the full build (Mac Zip = the only macOS smoke coverage). + +### D8. macOS runners: scheduled/built runs only, never per-commit + +`test-smokes-parallel.yml` (per-commit) must stay fast, so it never passes `runners` and keeps the `ubuntu-latest`/`windows-latest` default. +The only `macos-latest` smoke job is the nightly Mac leg in `test-smokes-built.yml`. +Encoded in the `runners` input description in `test-smokes.yml`. + +### D9. Built mode runs smoke + playwright + ff-matrix daily; the dev crons stay + +**Revised 2026-07-20** (originally: `integration/` stays dev-only with a future dev daily job — that deferral is resolved the other way). + +Built mode is not smoke-only: the nightly path (and every other source mode) runs three legs — smoke, playwright (`integration/playwright-tests.test.ts`), and the feature-format matrix — so packaging/launcher/signing regressions surface in browser behavior and the full ff corpus too, not just the smoke suite (see "Built-mode test legs"). +Both suites were only ever excluded from binary mode by the `run-tests.[sh|ps1]` default, never hard-blocked; the harness prerequisites (the `quartoSpawnEnvOptions()` render-env fix, playwright provisioning in non-dev CI modes) are in place. + +What stays dev-only: `unit/` (in-process by definition), the non-playwright `integration/` tests (`guess-chunk-options-format-document.test.ts`, `mermaid/github-issue-1340.test.ts` — dev shards only), `QUARTO_DEBUG` paths, the `quarto check` dev branch, and in-process races. +Also *temporarily* dev-only: the julia-engine subtree tests (`smoke/julia-engine/`, copied in by `merge-extension-tests`) — their raw `Deno.Command("quarto")` spawns inherit the harness dev env (the D3/D4 trap: `QUARTO_DEBUG` crashes the built quarto in `checkReconfiguration`), so the merge action skips them when `QUARTO_TEST_BIN` is set until the spawns are sanitized in `PumasAI/quarto-julia-engine`. +Residual gaps no suite exercises against the built quarto (not covered anywhere in CI today, recorded so they read as known boundaries rather than oversights): `quarto preview`/serve interactive paths (the playwright render glob excludes `docs/playwright/(serve|shiny)`), `quarto publish` flows (credentials), the actual installer packages (`.deb`/`.msi`/`.pkg` — the legs test the tarball/zip layouts, never install-time behavior like PATH or registry), the linux-arm64 tarball, and playwright visual snapshots (`--ignore-snapshots`). +Windows browser behavior and macOS ff-matrix are also uncovered but deliberate, with revisit conditions in "Built-mode test legs". +The daily dev crons also stay — dev ff-matrix catches source regressions, built ff-matrix catches packaging regressions; complementary, not redundant. +Nothing is throttled initially (all legs daily): per-leg OS scope lives in the scheduler jobs as the tuning knob once real CI spend is observed. +Known cost blind spot, accepted: the `workflow_run` trigger fires on EVERY completed create-release run (D1), so manual builds also get the full fan-out; gate the heavy legs on `github.event.workflow_run.event == 'schedule'` if that ever needs trimming. + +### D10. Release mode only works for post-harness tags + +Release mode checks out the tag, so the harness at that tag must already contain `tests/quarto-cmd.ts` — a preflight fails clearly for older releases. +True backfill (main-branch harness testing an older binary) would require harness/binary decoupling (plan §6 Phase 4, not implemented). + +The same skew applies per-suite: nightly/release legs run the harness at the *target* ref, so a ref that has `tests/quarto-cmd.ts` but predates the `quartoSpawnEnvOptions()` render fix in `playwright-tests.test.ts` (#14706) runs the old env-leaking wrapper — its playwright leg renders with dev-tree resources and its result (green or red) is not meaningful. +The existence preflight cannot detect this. +Affected window: releases and nightly build shas cut between the harness-support merge and the multi-leg merge, including the first post-merge `workflow_run` firings on pre-merge build commits. +The smoke and ff-matrix legs are unaffected (their spawns go through `runQuarto`, whose env sanitization is as old as `quarto-cmd.ts`). + +### D11. The automatic nightly leg tests an *unsigned* `quarto.exe` + +`create-release.yml` gates both DigiCert steps ("Sign files before making ZIP and MSI installer", "Sign MSI installer") on `github.event_name != 'schedule'`, so the daily scheduled build — the one `workflow_run` fires on every night — packages an unsigned `quarto.exe` into `Windows Zip`. +macOS is unaffected: `make-installer-mac` signs and notarizes on every event. + +This is sufficient for the packaging and launcher checks in binary mode. +The daily run exercises the real launcher (`package/launcher` `quarto.exe`) rather than the dev `.cmd` shim. +Signing changes the bytes, not the launcher's argument handling or resource resolution. + +Signed Windows coverage therefore comes from `create-release` *dispatches*, which sign and which `workflow_run` also tests (see D7) — the intended pre-merge check for bundled-binary bumps. +The scheduled path does not validate signing. diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index 5dc9e9d2aae..c469fa95fe3 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -1,3 +1,13 @@ +--- +main_commit: e5850df75 +analyzed_date: 2026-09-10 +key_files: + - tests/test.ts + - tests/quarto-cmd.ts + - tests/verify.ts + - tests/utils.ts +--- + # Quarto Test Patterns This document describes the standard patterns for writing smoke tests in the Quarto CLI test suite. @@ -7,9 +17,29 @@ This document describes the standard patterns for writing smoke tests in the Qua Quarto uses Deno for testing with custom verification helpers located in: - `tests/test.ts` - Core test runner (`testQuartoCmd`) +- `tests/quarto-cmd.ts` - Quarto invocation dispatch (`runQuarto`: in-process dev quarto vs built binary) - `tests/verify.ts` - Verification helpers (`fileExists`, `pathDoNotExists`, etc.) - `tests/utils.ts` - Utility functions (`docs()`, `outputForInput()`, etc.) +### Dev Mode vs Binary Mode + +`testQuartoCmd` does not call quarto directly — it goes through `runQuarto()` in `tests/quarto-cmd.ts`, the single dispatch point for invoking the quarto under test: + +- **Dev mode (default):** Quarto runs in-process via the `quarto()` entry point imported from `src/quarto.ts`. +- **Binary mode:** when `QUARTO_TEST_BIN` points at an installed quarto (a built distribution extracted *outside* this checkout), quarto is spawned as a subprocess with `--log --log-format json-stream`, so the log-record verifiers work unchanged. + Dev-tree env vars (`QUARTO_SHARE_PATH`, `QUARTO_DEBUG`, `DENO_DIR`, ...) + are stripped from the child. + `run-tests.sh`/`.ps1` refuse a binary reporting the `99.9.9` dev sentinel and default the selection to `smoke/` (`unit/` is dev-only; the playwright suite and ff-matrix corpus are binary-compatible and run when passed explicitly). + Exercised by `.github/workflows/test-smokes-built.yml`, which runs smoke + playwright + ff-matrix legs per source mode. + Architecture and design decisions: `llm-docs/built-version-testing-architecture.md`. + +Consequences for writing smoke tests: + +- Do **not** import `src/quarto.ts` (or call `quarto()`) directly from `tests/smoke/` — route invocations through `testQuartoCmd`/`runQuarto` so the test works in both modes. +- Tests that spawn a quarto subprocess themselves should resolve the executable via `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) and pass `quartoSpawnEnvOptions()` from `tests/quarto-cmd.ts` as spawn env options. +- A test that genuinely exercises quarto internals in-process can set `TestContext.requiresDevQuarto: true`; it is ignored in binary mode. + Use sparingly — most such code belongs in `tests/unit/` instead. + ## Common Test Patterns ### Simple Render Tests @@ -91,7 +121,8 @@ testQuartoCmd( ### Performance Budget (Render Timeout) -`testQuartoCmd` runs the render under a default 10-minute timeout. For a test guarding a *performance* regression — a render that must not hang — set a tight budget via `TestContext.timeout` (milliseconds) so a regression fails fast instead of riding the 10-minute default: +`testQuartoCmd` runs the render under a default 10-minute timeout. +For a test guarding a *performance* regression — a render that must not hang — set a tight budget via `TestContext.timeout` (milliseconds) so a regression fails fast instead of riding the 10-minute default: ```typescript testQuartoCmd("render", [projectDir], [noErrors /*, ... */], { @@ -109,8 +140,10 @@ testQuartoCmd("render", [projectDir], [noErrors /*, ... */], { **Key points:** -- The budget is machine-dependent (post-fix render time must sit well under it, pre-fix hang well over it), so it is defense-in-depth. Pair it with a deterministic unit test on the actual fix mechanism as the primary guard. -- A timed-out render subprocess is not killed by the harness, so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. +- The budget is machine-dependent (post-fix render time must sit well under it, pre-fix hang well over it), so it is defense-in-depth. + Pair it with a deterministic unit test on the actual fix mechanism as the primary guard. +- In dev (in-process) mode a timed-out render is not killed by the harness (the timeout only rejects), so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. + In binary mode (`QUARTO_TEST_BIN`) the spawned process tree *is* killed on timeout, but the kill is best-effort — keep the same defensive teardown. ### Extension Template Tests @@ -171,11 +204,9 @@ testQuartoCmd( ### Working-Directory-Sensitive Tests -Some tests need to run from a specific directory (e.g. reproducing a bug that -depends on the process cwd). **Do not `Deno.chdir()` inside the test body** — -it mutates process-global cwd and can leak into other tests in the same -process. Use the `TestContext` options instead; the harness changes the cwd -before the test and restores it afterward: +Some tests need to run from a specific directory (e.g. reproducing a bug that depends on the process cwd). +**Do not `Deno.chdir()` inside the test body** — it mutates process-global cwd and can leak into other tests in the same process. +Use the `TestContext` options instead; the harness changes the cwd before the test and restores it afterward: ```typescript const workingDir = Deno.makeTempDirSync(); @@ -191,17 +222,11 @@ unitTest("runs from workingDir", async () => { **Key points:** -- The harness calls `cwd()` **before** `setup()`, so the directory must already - exist when `cwd()` runs — create it at module scope, not in `setup`. -- `teardown` runs **before** the harness restores the cwd, so on Windows the - temp dir may still be the cwd and resist removal. Wrap the removal in - try/catch (best-effort) — see `tests/smoke/use/template.test.ts` and - `tests/unit/dotenv-config.test.ts`. -- For a temp directory you don't need to run *from*, prefer `withTempDir` - (`tests/utils.ts`), which creates and recursively removes it in a `finally`. -- A test that only needs a **relative** input (not a specific cwd) can pass a - path relative to the current cwd (`relative(Deno.cwd(), absFile)`) without - changing directories at all. +- The harness calls `cwd()` **before** `setup()`, so the directory must already exist when `cwd()` runs — create it at module scope, not in `setup`. +- `teardown` runs **before** the harness restores the cwd, so on Windows the temp dir may still be the cwd and resist removal. + Wrap the removal in try/catch (best-effort) — see `tests/smoke/use/template.test.ts` and `tests/unit/dotenv-config.test.ts`. +- For a temp directory you don't need to run *from*, prefer `withTempDir` (`tests/utils.ts`), which creates and recursively removes it in a `finally`. +- A test that only needs a **relative** input (not a specific cwd) can pass a path relative to the current cwd (`relative(Deno.cwd(), absFile)`) without changing directories at all. ## Verification Helpers @@ -356,7 +381,8 @@ See `tests/smoke/use/template.test.ts` for extension template patterns. - **Python**: `tests/.venv/` (managed by uv/pyproject.toml) - **R**: `tests/renv/` + `tests/renv.lock` -The `configure-test-env` scripts ONLY manage these main environments. CI builds depend on this structure. +The `configure-test-env` scripts ONLY manage these main environments. +CI builds depend on this structure. **Do NOT create language environment files in test subdirectories:** @@ -395,42 +421,34 @@ cd tests Rscript -e "renv::install(); renv::snapshot()" ``` -**Note:** While Quarto supports local Project.toml files in document directories for production use, the quarto-cli test infrastructure specifically does NOT support this pattern. All test dependencies must be in the main `tests/` environment. +**Note:** While Quarto supports local Project.toml files in document directories for production use, the quarto-cli test infrastructure specifically does NOT support this pattern. +All test dependencies must be in the main `tests/` environment. ### R Tests That Change Working Directory -R resolves `.Rprofile` from the **exact** process cwd (no parent-directory -search). On CI, rmarkdown/knitr live only in `tests/renv`'s project library, -activated when cwd is `tests/` (via `tests/.Rprofile` sourcing -`renv/activate.R`). Most knitr tests never leave `tests/` — they pass paths -relative to the current cwd instead of changing directories — so activation -happens automatically. - -A test that must run with cwd set elsewhere (a scratch temp dir, via -`TestContext.cwd()` — see "Working-Directory-Sensitive Tests" above) loses -that activation: the R subprocess starts outside `tests/`, renv never -activates, and package loads fail with `there is no package called -'rmarkdown'`. This is CI-only — a developer machine with rmarkdown on the -default `.libPaths()` masks it entirely. The render pipeline also tends to -swallow the underlying subprocess error, so the failure can be silent beyond -the bare package-load message. - -**Fix:** in the fixture's cwd, write a `.Rprofile` that re-points renv at the -real project, regardless of where the test's cwd actually is: +R resolves `.Rprofile` from the **exact** process cwd (no parent-directory search). +On CI, rmarkdown/knitr live only in `tests/renv`'s project library, activated when cwd is `tests/` (via `tests/.Rprofile` sourcing `renv/activate.R`). +Most knitr tests never leave `tests/` — they pass paths relative to the current cwd instead of changing directories — so activation happens automatically. + +A test that must run with cwd set elsewhere (a scratch temp dir, via `TestContext.cwd()` — see "Working-Directory-Sensitive Tests" above) loses that activation: the R subprocess starts outside `tests/`, renv never activates, and package loads fail with `there is no package called +'rmarkdown'`. +This is CI-only — a developer machine with rmarkdown on the default `.libPaths()` masks it entirely. +The render pipeline also tends to swallow the underlying subprocess error, so the failure can be silent beyond the bare package-load message. + +**Fix:** in the fixture's cwd, write a `.Rprofile` that re-points renv at the real project, regardless of where the test's cwd actually is: ```r Sys.setenv(RENV_PROJECT = "") source("/renv/activate.R") ``` -`renv/activate.R` reads `RENV_PROJECT` to determine the project root if set, -falling back to `getwd()` otherwise — setting it explicitly decouples renv -activation from the test's cwd. +`renv/activate.R` reads `RENV_PROJECT` to determine the project root if set, falling back to `getwd()` otherwise — setting it explicitly decouples renv activation from the test's cwd. ## Best Practices 1. **Always clean up**: Use teardown to remove generated files -2. **Use helpers**: Leverage `docs()`, `fileExists()`, etc. instead of manual checks +2. **Use helpers**: Leverage `docs()`, `fileExists()`, etc. + instead of manual checks 3. **Absolute paths**: Use `join()` for all path construction to handle platform differences 4. **Test isolation**: Use temp directories for tests that create files 5. **Clear names**: Use descriptive variable names like `projectDir`, `outputDir`, `templateFolder` @@ -439,23 +457,21 @@ activation from the test's cwd. ## Environment Variable Testing Pitfalls -`Deno.env.set()` modifies process-global state. Deno runs test files in parallel by default (same OS process), so concurrent tests can see modified values. Save/restore patterns don't help - other tests see the modified value during the test window. +`Deno.env.set()` modifies process-global state. +Deno runs test files in parallel by default (same OS process), so concurrent tests can see modified values. +Save/restore patterns don't help - other tests see the modified value during the test window. | Execution Mode | Risk | Why | | -------------------------- | ------------------ | --------------------------------------- | | `./run-tests.sh` (default) | **Race condition** | Files run in parallel, share `Deno.env` | | `./run-parallel-tests.sh` | **None** | Separate OS processes | -**Existing bad pattern** - `tests/smoke/website/drafts-env.test.ts`: +**Preferred channel:** pass per-test env via `TestContext.env` — it reaches the in-process `quarto()` call in dev mode and the spawned binary in binary mode (`QUARTO_TEST_BIN`), without mutating process-global state. -```typescript -// BAD: Sets env var, never restores it -// Only "works" because no other test reads QUARTO_PROFILE -Deno.env.set("QUARTO_PROFILE", "drafts"); -testQuartoCmd("render", [renderDir], [...]); -``` +**Known justified exception** - `tests/smoke/website/drafts-env.test.ts` still sets `QUARTO_PROFILE` at module load *in addition to* `context.env`: `src/project/project-profile.ts` caches the base profile from the env on the first render in the process (`baseQuartoProfile`), so in dev (in-process) mode a per-render env override is ignored whenever another test rendered first. +The module-load set runs before any test and keeps the cache correct; the `context.env` copy is what the spawned binary sees in binary mode. -**Alternatives:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. +**Alternatives for new tests:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. ## Testing File Exclusion @@ -479,7 +495,8 @@ Run test **without fix** first to verify it fails, then verify it passes with fi ## Smoke-All Tests (YAML-Based) -Smoke-all tests embed test specifications directly in `.qmd` files using `_quarto.tests` metadata. See `.claude/rules/testing/smoke-all-tests.md` for full documentation. +Smoke-all tests embed test specifications directly in `.qmd` files using `_quarto.tests` metadata. +See `.claude/rules/testing/smoke-all-tests.md` for full documentation. ### YAML String Escaping for Regex @@ -511,33 +528,25 @@ _quarto: ### Probe enough keys to surface the bug -A precedence test where the template reads only the one key being -overridden can pass under a deep-merge bug. The dropped sibling keys -never resolve, but no assertion notices. +A precedence test where the template reads only the one key being overridden can pass under a deep-merge bug. +The dropped sibling keys never resolve, but no assertion notices. -Example: the merge of user-supplied `variables.quarto.language.crossref-ch-prefix: Bouquin` -onto Quarto's built `format.language` table under -`variables.quarto.language`. Under a shallow spread (`{ ...a, ...b }`), -`b.language` replaces the entire localized map — all other -`$quarto.language.$` resolutions silently return empty. A template -that reads only `$quarto.language.crossref-ch-prefix$` still asserts -"Bouquin", so the regression test passes. +Example: the merge of user-supplied `variables.quarto.language.crossref-ch-prefix: Bouquin` onto Quarto's built `format.language` table under `variables.quarto.language`. +Under a shallow spread (`{ ...a, ...b }`), `b.language` replaces the entire localized map — all other `$quarto.language.$` resolutions silently return empty. +A template that reads only `$quarto.language.crossref-ch-prefix$` still asserts "Bouquin", so the regression test passes. -The fix is to probe at least one non-overridden sibling key in the same -template. Concretely, the regression guard -`tests/docs/smoke-all/markdown/lang-fr-user-override-deep-merge.qmd` -uses the template +The fix is to probe at least one non-overridden sibling key in the same template. +Concretely, the regression guard `tests/docs/smoke-all/markdown/lang-fr-user-override-deep-merge.qmd` uses the template ``` $quarto.language.crossref-ch-prefix$|$quarto.language.toc-title-document$ ``` -and asserts the full string `^Bouquin\|Table des matières\s*$`. Pre-fix -the output was `Bouquin|`; post-fix it is `Bouquin|Table des matières`. +and asserts the full string `^Bouquin\|Table des matières\s*$`. +Pre-fix the output was `Bouquin|`; post-fix it is `Bouquin|Table des matières`. -Heuristic: when writing a precedence smoke test for any merge between -two structured config trees, ensure the assertion exercises at least -one path the user did NOT override. Otherwise the test only proves -"the overridden value wins" — not "the rest survives". +Heuristic: when writing a precedence smoke test for any merge between two structured config trees, ensure the assertion exercises at least one path the user did NOT override. +Otherwise the test only proves "the overridden value wins" — not "the rest survives". -**Recommendation:** Use single-quoted strings. They're simpler - only `'` itself needs escaping (as `''`). +**Recommendation:** Use single-quoted strings. +They're simpler - only `'` itself needs escaping (as `''`). diff --git a/tests/README.md b/tests/README.md index ef401eddaad..dd0a5a119c2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,7 +12,8 @@ Tests are run in our CI workflow on GHA at each commit, and for each PR. ## How the tests are created and organized ? -Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. Infrastructure are in `tests.ts`, `tests.deps.ts` `verify.ts` and `utils.ts` which contains the helper functions that can be used. +Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. +Infrastructure are in `test.ts`, `test-deps.ts`, `quarto-cmd.ts`, `verify.ts` and `utils.ts` which contains the helper functions that can be used. - `unit/` and `integration/`, `smoke/`contain some `.ts` script representing each tests. - `docs/` is a special folder containing of the necessary files and projects used for the tests. @@ -23,21 +24,27 @@ Tests are running through `Deno.test()` framework, adapted for our Quarto projec Here are what is expected in the environment for the tests : -- R should be installed and in PATH - [**rig**](https://github.com/r-lib/rig) is a good tool to manage R versions. e.g `rig install 4.4.2` and `rig default 4.4.2` to install and set the version to 4.4.2 +- R should be installed and in PATH - [**rig**](https://github.com/r-lib/rig) is a good tool to manage R versions. + e.g `rig install 4.4.2` and `rig default 4.4.2` to install and set the version to 4.4.2 - On Windows, Rtools should be too (for source package installation) - Python should be installed and in PATH - [**pyenv**](https://github.com/pyenv/pyenv) is a good option to manage Python versions. - - On Windows, it will be [`pyenv-win`](https://pyenv-win.github.io/pyenv-win/) to manage versions. Otherwise or install from https://www.python.org/ manually or using `winget`. + - On Windows, it will be [`pyenv-win`](https://pyenv-win.github.io/pyenv-win/) to manage versions. + Otherwise or install from https://www.python.org/ manually or using `winget`. - Julia should be installed and in PATH - [**juliaup**](https://github.com/JuliaLang/juliaup) is a good option to manage Julia versions. - On Windows, one way is using `winget install julia -s msstore` and then add `%LOCALAPPDATA%/Programs/Julia/bin` to PATH Running tests require to have a local environment setup with Quarto development, TinyTeX, R, Python and Julia. -To help with this configuration, the `tests/` folder contains `configure-test-env.sh` and `configure-test-env.ps1`. It will check for the tools and update the dependencies to what is used by Quarto tests. -Running the script at least one will insure you are correctly setup. Then, it is run as part of running the tests so that dependencies are always updated. Set `QUARTO_TESTS_NO_CONFIG` to skip this step when running tests. +To help with this configuration, the `tests/` folder contains `configure-test-env.sh` and `configure-test-env.ps1`. +It will check for the tools and update the dependencies to what is used by Quarto tests. +Running the script at least one will insure you are correctly setup. +Then, it is run as part of running the tests so that dependencies are always updated. +Set `QUARTO_TESTS_NO_CONFIG` to skip this step when running tests. #### Optional test dependencies -The configure scripts also check for optional tools that some tests require. Tests will gracefully skip when these tools are not available, but having them installed enables full test coverage: +The configure scripts also check for optional tools that some tests require. +Tests will gracefully skip when these tools are not available, but having them installed enables full test coverage: **Java** (version 8, 11, 17, or 21) @@ -77,11 +84,15 @@ Dependencies are managed using the following tools: #### R -We use [**renv**](https://rstudio.github.io/renv/). `renv.lock` and `renv/` folders are the files used to recreate the environment for R. +We use [**renv**](https://rstudio.github.io/renv/). +`renv.lock` and `renv/` folders are the files used to recreate the environment for R. -Updating `renv.lock` is done using `renv::snapshot()`. File shouldn't be modified manually. +Updating `renv.lock` is done using `renv::snapshot()`. +File shouldn't be modified manually. -Our project is using [explicit dependencies discovery](https://rstudio.github.io/renv/reference/dependencies.html?q=dependen#explicit-dependencies) through a `DESCRIPTION` file. This is to avoid a costly scanning of all files in `tests/` to guess R dependencies. This means that if you need to add a test with a new R package dependencies: +Our project is using [explicit dependencies discovery](https://rstudio.github.io/renv/reference/dependencies.html?q=dependen#explicit-dependencies) through a `DESCRIPTION` file. +This is to avoid a costly scanning of all files in `tests/` to guess R dependencies. +This means that if you need to add a test with a new R package dependencies: - Add package(s) to `DESCRIPTION` in `tests/` - `renv::install()` the package into the project library @@ -91,26 +102,36 @@ Our project is using [explicit dependencies discovery](https://rstudio.github.io See [documentation](https://rstudio.github.io/renv/) if you need to tweak the R environment. -After a dependency update, you can run `configure-test-env.sh` or `configure-test-env.ps1` to update the environment, or manually run `renv::restore()` to recreate the environment with new versions. Be sure to update your R version if needed. +After a dependency update, you can run `configure-test-env.sh` or `configure-test-env.ps1` to update the environment, or manually run `renv::restore()` to recreate the environment with new versions. +Be sure to update your R version if needed. #### Python -We now use [**uv**](https://docs.astral.sh/uv) (previously, it was [**pipenv**](https://pipenv.pypa.io/en/latest/)) to manage dependencies and recreate easily on all OS. `uv` will **not** be installed as part of the configuration - so it needs to be installed manually - see various way at: https://docs.astral.sh/uv/getting-started/installation/ +We now use [**uv**](https://docs.astral.sh/uv) (previously, it was [**pipenv**](https://pipenv.pypa.io/en/latest/)) to manage dependencies and recreate easily on all OS. +`uv` will **not** be installed as part of the configuration - so it needs to be installed manually - see various way at: https://docs.astral.sh/uv/getting-started/installation/ -**uv** will handle the python versions, including its installation, based on the `.python-version` we have in `tests/` folder. It will also manage the virtual environment in `.venv` folder. +**uv** will handle the python versions, including its installation, based on the `.python-version` we have in `tests/` folder. +It will also manage the virtual environment in `.venv` folder. -A virtual environment will be created locally in `.venv` folder (ignored on git) and activated when running tests. `uv run` can help activating the environment outside of running tests to run a command in the environment. +A virtual environment will be created locally in `.venv` folder (ignored on git) and activated when running tests. +`uv run` can help activating the environment outside of running tests to run a command in the environment. -`pyproject.toml` contains our dependencies requirement for the tests project. It can be manually updated but it is best to just use `uv` commands. For instance, adding a new dependency can be done with `uv add plotly` and it will update the file, update the `uv.lock` and install in the virtual environment. `uv.lock` should never be updated manually, and it is tracked by git, as it allows to recreate the exact environment on different environment (Linux, Mac, Windows, locally and on CI). +`pyproject.toml` contains our dependencies requirement for the tests project. +It can be manually updated but it is best to just use `uv` commands. +For instance, adding a new dependency can be done with `uv add plotly` and it will update the file, update the `uv.lock` and install in the virtual environment. +`uv.lock` should never be updated manually, and it is tracked by git, as it allows to recreate the exact environment on different environment (Linux, Mac, Windows, locally and on CI). See other [`uv` command](https://docs.astral.sh/uv/getting-started/features/) if you need to do more. -For a change of python versionn, `.python-version` needs to be updated, and then `uv` will take care of the rest. `configure-test-env` script will check for `uv` and if installed, it will called `uv sync` to make sure the project virtual environment is up to date with the lockfile. +For a change of python versionn, `.python-version` needs to be updated, and then `uv` will take care of the rest. +`configure-test-env` script will check for `uv` and if installed, it will called `uv sync` to make sure the project virtual environment is up to date with the lockfile. Note that `./run-test.ps1` and `.run-tests.sh` : - run `configure-test-env` script by default, unless `QUARTO_TESTS_NO_CONFIG` environment variable is set to a non-empty value. -- Activate the local virtualenv espected in `.venv`. Set `QUARTO_TESTS_FORCE_NO_VENV` to a non-empty value to prevent this behavior. (It replaces `QUARTO_TESTS_FORCE_NO_PIPENV` which still is considered for backward compatibility but deprecated) +- Activate the local virtualenv espected in `.venv`. + Set `QUARTO_TESTS_FORCE_NO_VENV` to a non-empty value to prevent this behavior. + (It replaces `QUARTO_TESTS_FORCE_NO_PIPENV` which still is considered for backward compatibility but deprecated) #### Julia @@ -118,7 +139,8 @@ Julia uses built-in package manager [**Pkg.jl**](https://pkgdocs.julialang.org/v `Project.toml` contains our direct dependency and `Manifest.toml` is the lock file that will be created (`Pkg.resolve()`). -**Important:** All test dependencies must be in the main `tests/` environment. Julia searches UP the directory tree for `Project.toml` starting from the document being rendered. +**Important:** All test dependencies must be in the main `tests/` environment. +Julia searches UP the directory tree for `Project.toml` starting from the document being rendered. **Adding a new package dependency:** @@ -128,9 +150,13 @@ julia --project=. -e 'using Pkg; Pkg.add("PackageName")' ./configure-test-env.sh # or .ps1 on Windows ``` -**Do NOT create** local `Project.toml` files in test subdirectories (e.g., `tests/docs/*/Project.toml`). Julia will use that environment instead of the main `tests/` environment. The `configure-test-env` scripts only manage the main environment, so tests with local environments will fail in CI even if they work locally. +**Do NOT create** local `Project.toml` files in test subdirectories (e.g., `tests/docs/*/Project.toml`). +Julia will use that environment instead of the main `tests/` environment. +The `configure-test-env` scripts only manage the main environment, so tests with local environments will fail in CI even if they work locally. -**Note:** This applies to ALL engines (Julia, Python, R). Python and R will also use local `.venv/` or `renv.lock` if present. The quarto-cli test infrastructure uses a single managed environment per language at `tests/`, and CI only configures these main environments. +**Note:** This applies to ALL engines (Julia, Python, R). +Python and R will also use local `.venv/` or `renv.lock` if present. +The quarto-cli test infrastructure uses a single managed environment per language at `tests/`, and CI only configures these main environments. See [documentation](https://pkgdocs.julialang.org/v1/managing-packages/) on how to add, remove, update if you need to tweak the Julia environment. @@ -198,7 +224,8 @@ For convenience, `run-fast-tests.sh` and `run-fast-tests.ps1` are provided to sk ./run-fast-tests.ps1 ``` -These scripts set `QUARTO_TESTS_NO_CONFIG` automatically. Use after running `configure-test-env` at least once. +These scripts set `QUARTO_TESTS_NO_CONFIG` automatically. +Use after running `configure-test-env` at least once. **QUARTO_TEST_KEEP_OUTPUTS** (or use `--keep-outputs`/`-k` flag) - Keep test output artifacts instead of cleaning them up @@ -230,7 +257,9 @@ $env:QUARTO_TEST_KEEP_OUTPUTS="true" #### About smoke-all tests -`docs/smoke-all/` is a specific folder to run some tests written directly within `.qmd`, `.md` or `.ipynb` files (but files starting with `_` will be ignored). They are run through the `smoke/smoke-all.tests.ts` script. To ease running smoke-all tests, `run-tests.sh` has a special behavior where it will run `./smoke/smoke-all.tests.ts` when passed a `.qmd`, `.md` or `.ipynb` file, not starting with `_`. +`docs/smoke-all/` is a specific folder to run some tests written directly within `.qmd`, `.md` or `.ipynb` files (but files starting with `_` will be ignored). +They are run through the `smoke/smoke-all.tests.ts` script. +To ease running smoke-all tests, `run-tests.sh` has a special behavior where it will run `./smoke/smoke-all.tests.ts` when passed a `.qmd`, `.md` or `.ipynb` file, not starting with `_`. ```bash # run tests for all documents in docs/smoke-all/ @@ -406,9 +435,12 @@ When a snapshot test fails: ### Limitations -- `smoke-all.test.ts` accept only one argument. You need to use glob pattern to run several smoke-all test documents. +- `smoke-all.test.ts` accept only one argument. + You need to use glob pattern to run several smoke-all test documents. -- Individual `smoke-all` tests and other test can't be run at the same time with `run-test.[sh|ps1]`. This is because `smoke-all.test.ts` requires arguments. If a smoke-all document and another smoke-test are passed as argument, the smoke-all test will be prioritize and other will be ignored (with a warning). +- Individual `smoke-all` tests and other test can't be run at the same time with `run-test.[sh|ps1]`. + This is because `smoke-all.test.ts` requires arguments. + If a smoke-all document and another smoke-test are passed as argument, the smoke-all test will be prioritize and other will be ignored (with a warning). Example with Linux: @@ -430,9 +462,59 @@ Don't do ./run-tests.sh smoke/extensions/extension-render-doc.test.ts smoke/smoke-all.test.ts -- ./docs/smoke-all/2023/01/04/issue-3847.qmd ``` +### Binary mode (`QUARTO_TEST_BIN`) + +Tests normally run Quarto in-process from the dev sources. +Set `QUARTO_TEST_BIN` to an installed Quarto to run commands against that binary instead. +See [Built-Version Testing Architecture](../llm-docs/built-version-testing-architecture.md) for the harness and CI design. + +To run in binary mode locally: + +```bash +# 1. Build a distribution (after ./configure.sh) +cd package/src +./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" +cd ../.. + +# 2. Copy the built dist OUTSIDE the git checkout. An in-repo quarto +# (e.g. package/dist/bin/quarto) resolves to dev mode — the launcher runs +# the TS sources when it finds a sibling src/quarto.ts — and must NOT be +# used: run-tests.[sh|ps1] refuses a binary reporting the 99.9.9 dev +# version sentinel. +cp -r package/pkg-working ~/quarto-under-test + +# 3. Run the tests against it +cd tests +QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh +``` + +In binary mode: + +- With no arguments, `run-tests.[sh|ps1]` runs `smoke/`. + Unit tests remain dev-only. + Playwright and feature-format tests support binary mode but must be passed explicitly: + + ```bash + # playwright suite against a built quarto + QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh integration/playwright-tests.test.ts + # feature-format matrix against a built quarto + QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" + ``` + +- The test environment is configured as usual; set `QUARTO_TESTS_NO_CONFIG` to skip that step as in dev mode. +- Tests with `requiresDevQuarto: true` in their `TestContext` are ignored (rare escape hatch for tests that must exercise quarto internals in-process). + +Authoring rules that keep tests working in both modes: + +- Never `import { quarto } from "../src/quarto.ts"` in tests — invoke quarto through `testQuartoCmd()` (`tests/test.ts`) or `runQuarto()` (`tests/quarto-cmd.ts`). +- Tests that spawn quarto as a subprocess themselves should resolve the executable with `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) — or `quartoDevBinCmd()` (`tests/quarto-cmd.ts`) when the test must pin the locally-built dev CLI — and pass `quartoSpawnEnvOptions()` as spawn env options so the dev-tree env vars don't leak into the built quarto. + +Harness and CI flow diagrams are in the architecture document linked above. + ## Debugging within tests -`.vscode/launch.json` has a `Run Quarto test` configuration that can be used to debug when running tests. One need to modify the `program` and `args` fields to match the test to run. +`.vscode/launch.json` has a `Run Quarto test` configuration that can be used to debug when running tests. +One need to modify the `program` and `args` fields to match the test to run. Example: @@ -451,7 +533,8 @@ This lives in `run-parallel-tests.ts` and called through `run-parallel-tests.sh` ### How does is works ? -- It requires a text file with tested timed and following a specific format. (Default is `timing.txt` and here is an example [in our repo](./timing.txt)) +- It requires a text file with tested timed and following a specific format. + (Default is `timing.txt` and here is an example [in our repo](./timing.txt)) - Based on this file, the tests will be split in buckets to minimize the tests time (buckets are filled by their minimum overall time). - Then `./run-tests.sh` will be run for each bucket from deno using `Promise.all()` and `run-tests.sh` on the whole bucket's test files, so that the buckets are ran in parallel. @@ -468,9 +551,11 @@ QUARTO_TEST_TIMING='timing.txt' ./run-tests.sh When this is done, any other argument will be ignored, and the following happens - All the `*.test.ts` file are found and run individually using `/usr/bin/time` to store timing in the file -- When `smoke-all.test.ts` is found, all the `*.qmd`, `*.md` and `*.ipynb` in `docs/smoke-all/` not starting with `_` are found and run individually using same logic. This means each `smoke-all` test is timed. +- When `smoke-all.test.ts` is found, all the `*.qmd`, `*.md` and `*.ipynb` in `docs/smoke-all/` not starting with `_` are found and run individually using same logic. + This means each `smoke-all` test is timed. -The results is written in the `$QUARTO_TEST_TIMING` file. Here is an example: +The results is written in the `$QUARTO_TEST_TIMING` file. +Here is an example: ``` ./smoke/directives/include-fixups.test.ts @@ -491,7 +576,8 @@ This will be read by `run-parallel-tests.ts` to get the `real` value and fill th #### Specific behavior for `smoke-all.test.ts` -`smoke-all` tests are special because they are in the form of individual `.qmd` or `.ipynb` document that needs to be run using `smoke-all.test.ts` script, with arguments. Unfortunately, this prevent running individual `smoke-all` documents in same buclets as other individual smoke test (which are their own `.test.ts` file). +`smoke-all` tests are special because they are in the form of individual `.qmd` or `.ipynb` document that needs to be run using `smoke-all.test.ts` script, with arguments. +Unfortunately, this prevent running individual `smoke-all` documents in same buclets as other individual smoke test (which are their own `.test.ts` file). So, if the timed file contains some individual timing for `smoke-all` documents like this @@ -499,16 +585,26 @@ So, if the timed file contains some individual timing for `smoke-all` documents ./smoke/smoke-all.test.ts -- docs/smoke-all/2022/12/12/code-annotation.qmd ``` -then they are ignored and `.smoke-all.test.ts` will be run in its own bucket. It will usually be the longest test run. +then they are ignored and `.smoke-all.test.ts` will be run in its own bucket. +It will usually be the longest test run. -Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tests on GHA CI as the buckets are split into their own runners and each test in a bucket if run using `run-test.sh`. This allows a bucket to contains some `*.test.ts` but also some document `*.qmd` or `*.ipynb`. More details in [test-smoke.yml](.github/workflows/test-smokes.yml) and [test-smokes-parallel.yml](.github/workflows/test-smokes-parallel.yml) +Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tests on GHA CI as the buckets are split into their own runners and each test in a bucket if run using `run-test.sh`. +This allows a bucket to contains some `*.test.ts` but also some document `*.qmd` or `*.ipynb`. +More details in [test-smoke.yml](.github/workflows/test-smokes.yml) and [test-smokes-parallel.yml](.github/workflows/test-smokes-parallel.yml) ### Arguments that control behavior -- `-n=`: Number of buckets to create to run in parallel. `run-parallel-tests.sh -n=5` split tests in 5 buckets and run them at the same time. For local run, `n` should be a number of core. For CI run, `n` will be the number of runners to use at the same time (mulplied by 2 because Linux and Windows are ran on CI). -- `--verbose`: show some verbosity. Otherwise, no specific logging in console in done. -- `--dry-run`: show the buckets of tests, but do not run. Otherwise, they are run. -- `--timing-file=`: Which file to use as timed tests information to creates the buckets. (default to `timing.txt` ). `run-parallel-tests.sh --timing-file='timing2.txt'` will use `timing2.txt` to run the file. +- `-n=`: Number of buckets to create to run in parallel. + `run-parallel-tests.sh -n=5` split tests in 5 buckets and run them at the same time. + For local run, `n` should be a number of core. + For CI run, `n` will be the number of runners to use at the same time (mulplied by 2 because Linux and Windows are ran on CI). +- `--verbose`: show some verbosity. + Otherwise, no specific logging in console in done. +- `--dry-run`: show the buckets of tests, but do not run. + Otherwise, they are run. +- `--timing-file=`: Which file to use as timed tests information to creates the buckets. + (default to `timing.txt` ). + `run-parallel-tests.sh --timing-file='timing2.txt'` will use `timing2.txt` to run the file. - `--json-for-ci`: Special flag to trigger splitting tests in buckets for the parallel run on CI and that makes `run-parallel-tests.sh` outputs JSON string specifically formatted for GHA processing. ### About tests in CI with GHA @@ -519,3 +615,15 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes - `test-smokes.yml` is the main CI workflow which configure the environment, and run the tests on Ubuntu and Windows. - If it was triggerred by `workflow_call`, then it will run each test in using `run-tests.[sh|ps1]` in a for-loop. - Scheduled tests are still run daily in their sequential version. + - It is parameterized (`quarto-install`, `quarto-version`, `quarto-artifact-name`, `ref`, `runners`, ...) so callers can run the suite against a built quarto instead of the dev source tree: the workflow installs the quarto under test outside the checkout and exports `QUARTO_TEST_BIN` (see "Binary mode" above). +- `test-smokes-built.yml` runs smoke, Playwright, and feature-format legs + against a built Quarto. It runs after nightly builds and supports manual + dispatches: + + | Mode | Trigger | Use it to answer | + |---|---|---| + | `nightly` | automatic after create-release; dispatch with `run-id` to retest an older run | does the packaged nightly build pass on each available OS? | + | `build` | dispatch (default) | does this ref work when packaged as a linux-amd64 distribution? | + | `release` | dispatch | does the published release pass? | + + Full rationale and design decisions: `llm-docs/built-version-testing-architecture.md`. diff --git a/tests/docs/convert/issue-12318.qmd b/tests/docs/convert/issue-12318.qmd new file mode 100644 index 00000000000..8859466ef00 --- /dev/null +++ b/tests/docs/convert/issue-12318.qmd @@ -0,0 +1,28 @@ +--- +title: Quarto Crossrefs +format: html +jupyter: python3 +categories: [plot, python, equation] +--- + +## Overview + +See @fig-simple in @sec-plot for a demonstration of a simple plot. + +See @eq-stddev to better understand standard deviation. + +## Plot {#sec-plot} + +```{python} +#| label: fig-simple +#| fig-cap: "Simple Plot" +import matplotlib.pyplot as plt +plt.plot([1,23,2,4]) +plt.show() +``` + +## Equation {#sec-equation} + +$$ +s = \sqrt{\frac{1}{N-1} \sum_{i=1}^N (x_i - \overline{x})^2} +$$ {#eq-stddev} diff --git a/tests/docs/crossrefs/editor-support/all.qmd b/tests/docs/crossrefs/editor-support/all.qmd new file mode 100644 index 00000000000..b027c14e316 --- /dev/null +++ b/tests/docs/crossrefs/editor-support/all.qmd @@ -0,0 +1,26 @@ +--- +title: All crossref types +--- + +## Introduction {#sec-intro} + +See @sec-intro for the overview, @fig-plot for the figure, @tbl-data for the +table, @eq-var for the equation, and @thm-line for the theorem. + +![A simple plot.](plot.png){#fig-plot} + +| Column A | Column B | +|----------|----------| +| 1 | 2 | + +: A small table. {#tbl-data} + +$$ +\sigma^2 = \frac{1}{N} \sum_{i=1}^N (x_i - \mu)^2 +$$ {#eq-var} + +::: {#thm-line} +## Line + +The equation of a straight line is $y = mx + b$. +::: diff --git a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml index eb86e8b6bf2..1a6f1d67ea7 100644 --- a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml +++ b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml @@ -1,7 +1,7 @@ title: Test author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: project: project: diff --git a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml index 2d5882388ab..66571285084 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml index 2d5882388ab..66571285084 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml index 859925eab79..c95827710de 100644 --- a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My Brand author: Quarto version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml index 106a96726ca..4bc546fbf56 100644 --- a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml +++ b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml @@ -1,7 +1,7 @@ title: Typst-brand-typography-example author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml index f11d93da35b..bbc62e89a3c 100644 --- a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml +++ b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml @@ -1,7 +1,7 @@ title: Font Provider Extension author: Test version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: formats: typst: diff --git a/tests/integration/playwright-tests.test.ts b/tests/integration/playwright-tests.test.ts index 4318d9836c0..dbf191095b3 100644 --- a/tests/integration/playwright-tests.test.ts +++ b/tests/integration/playwright-tests.test.ts @@ -1,5 +1,5 @@ /* - * smoke-all.test.ts + * playwright-tests.test.ts * * Copyright (C) 2022 Posit Software, PBC * @@ -13,6 +13,7 @@ import { } from "../../src/core/lib/yaml-validation/state.ts"; import { cleanoutput } from "../smoke/render/render.ts"; import { execProcess } from "../../src/core/process.ts"; +import { quartoSpawnEnvOptions } from "../quarto-cmd.ts"; import { quartoDevCmd } from "../utils.ts"; import { fail } from "testing/asserts"; import { isWindows } from "../../src/deno_ral/platform.ts"; @@ -72,11 +73,13 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { // mediabag inspection if we don't wait all renders // individually. This is very slow.. console.log(`Rendering ${input}...`); + // Prevent a built Quarto from inheriting dev-tree paths. const result = await execProcess({ cmd: quartoDevCmd(), args: ["render", input, ...options], stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); if (!result.success) { @@ -91,8 +94,8 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { } Deno.test({ - name: "Playwright tests are passing", - // currently we run playwright tests only on Linux + name: "Playwright tests are passing", + // Windows CI renders the inputs but does not run browser assertions. ignore: gha.isGitHubActions() && isWindows, fn: async () => { try { diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts new file mode 100644 index 00000000000..4e829b42ad5 --- /dev/null +++ b/tests/quarto-cmd.ts @@ -0,0 +1,429 @@ +/* +* quarto-cmd.ts +* +* Single dispatch point for invoking the quarto under test. +* +* Runs the dev sources in-process or QUARTO_TEST_BIN as a subprocess. +* See llm-docs/built-version-testing-architecture.md. +* +* Copyright (C) 2020-2026 Posit Software, PBC +* +*/ +import { quarto } from "../src/quarto.ts"; +import { kLocalDevelopment } from "../src/core/quarto.ts"; +import { isWindows } from "../src/deno_ral/platform.ts"; +import { join } from "../src/deno_ral/path.ts"; + +// Strip dev-tree and logging state from built-binary spawns. Other ambient +// variables are inherited, then the per-test environment is applied. +const kStripEnvVars = [ + "QUARTO_SHARE_PATH", + "QUARTO_BIN_PATH", + "QUARTO_DEBUG", + "DENO_DIR", + "QUARTO_DENO", + "QUARTO_DENO_DOM", + "QUARTO_ROOT", + "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION", + "QUARTO_VERSION_REQUIREMENT", + "QUARTO_PROJECT_DIR", + "QUARTO_PROFILE", + "QUARTO_LOG", + "QUARTO_LOG_LEVEL", + "QUARTO_LOG_FORMAT", + "RSTUDIO", +]; + +// std/log LogLevels.ERROR, as expected by readExecuteOutput(). +const kErrorLevel = 40; + +// Default per-invocation render timeout (dev and binary mode alike). +const kDefaultRenderTimeoutMs = 600000; + +// Path of the built quarto under test, when binary mode is active. +export function quartoTestBin(): string | undefined { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + return bin && bin.length > 0 ? bin : undefined; +} + +// True when tests target an external built Quarto. +export function isBinaryMode(): boolean { + return quartoTestBin() !== undefined; +} + +// Use QUARTO_TEST_BIN in binary mode; otherwise pin the local dev CLI. +export function quartoDevBinCmd(): string { + const bin = quartoTestBin(); + if (bin) { + return bin; + } + const binPath = Deno.env.get("QUARTO_BIN_PATH") ?? + join("..", "package", "dist", "bin"); + return join(binPath, isWindows ? "quarto.cmd" : "quarto"); +} + +export function buildBinaryEnv( + overlay?: Record, +): Record { + const env = Deno.env.toObject(); + for (const name of kStripEnvVars) { + delete env[name]; + } + return { ...env, ...(overlay ?? {}) }; +} + +// Sanitize direct subprocess spawns in binary mode. Dev-mode spawns inherit +// the ambient environment and apply only the requested overlay. +export function quartoSpawnEnvOptions( + overlay?: Record, +): { env?: Record; clearEnv?: boolean } { + if (isBinaryMode()) { + return { env: buildBinaryEnv(overlay), clearEnv: true }; + } + return overlay !== undefined ? { env: overlay } : {}; +} + +// Appends a synthetic ERROR record to a json-stream log file. Only call +// after the child process has exited (single-writer at that point). +export function appendLogError(logFile: string, msg: string) { + const record = JSON.stringify({ + msg, + level: kErrorLevel, + levelName: "ERROR", + }); + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // file may not exist yet + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + record + "\n"); +} + +// A timeout can interrupt a log write. Remove only the torn trailing record +// so readExecuteOutput() can remain strict. +function stripTornTrailingLine(content: string): string { + const lines = content.split("\n"); + let i = lines.length - 1; + while (i >= 0 && lines[i] === "") { + i--; + } + if (i < 0) { + return content; + } + try { + JSON.parse(lines[i]); + return content; + } catch { + lines.splice(i, 1); + return lines.join("\n"); + } +} + +function hasErrorRecordText(content: string): boolean { + for (const line of content.split("\n")) { + if (!line) continue; + try { + const record = JSON.parse(line); + if ( + typeof record?.levelName === "string" && + record.levelName.toLowerCase() === "error" + ) { + return true; + } + } catch { + // tolerate partial/corrupt lines + } + } + return false; +} + +// Reject an in-checkout launcher, which reports the 99.9.9 dev sentinel. +let checkedBinary: string | undefined; +export function assertTestBinary(bin: string) { + if (checkedBinary === bin) { + return; + } + // Probe with the same sanitized environment used by test spawns. + const result = new Deno.Command(bin, { + args: ["--version"], + stdout: "piped", + stderr: "piped", + env: buildBinaryEnv(), + clearEnv: true, + }).outputSync(); + const version = new TextDecoder().decode(result.stdout).trim(); + if (result.code !== 0) { + const stderr = new TextDecoder().decode(result.stderr).trim(); + throw new Error( + `QUARTO_TEST_BIN (${bin}) failed to report a version (exit ${result.code}):\n${stderr}`, + ); + } + if (version.length === 0) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reported an empty version. ` + + `The distribution is likely incomplete (missing share/version).`, + ); + } + if (version === kLocalDevelopment) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kLocalDevelopment}. ` + + `It is resolving to a dev-mode quarto (the launcher runs the TS sources when a ` + + `sibling src/quarto.ts exists). Point QUARTO_TEST_BIN at a built distribution ` + + `extracted outside the git checkout.`, + ); + } + const expected = Deno.env.get("QUARTO_TEST_EXPECTED_VERSION"); + if (expected && version !== expected) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports version ${version}, expected ${expected} ` + + `(QUARTO_TEST_EXPECTED_VERSION).`, + ); + } + console.log(`[binary mode] testing quarto ${version} at ${bin}`); + checkedBinary = bin; +} + +// The launcher waits on Deno, so kill the process tree deepest first. +async function killProcessTree(pid: number) { + if (isWindows) { + let killed = false; + try { + // taskkill reports failure through its exit code. + const result = await new Deno.Command("taskkill", { + args: ["/PID", String(pid), "/T", "/F"], + stdout: "null", + stderr: "null", + }).output(); + killed = result.code === 0; + } catch { + // Fall through to a direct kill. + } + if (!killed) { + // Ensure child.output() can resolve even if the tree kill failed. + try { + Deno.kill(pid, "SIGKILL"); + } catch { + // already exited + } + } + return; + } + const pids: number[] = []; + const stack = [pid]; + while (stack.length > 0) { + const current = stack.pop()!; + pids.push(current); + try { + // pgrep -P works on Linux and macOS/BSD. + const result = new Deno.Command("pgrep", { + args: ["-P", String(current)], + stdout: "piped", + stderr: "null", + }).outputSync(); + const children = new TextDecoder() + .decode(result.stdout) + .split("\n") + .map((line) => parseInt(line.trim(), 10)) + .filter((child) => !isNaN(child)); + stack.push(...children); + } catch { + // pgrep unavailable; fall back to killing what we have + } + } + for (const target of pids.reverse()) { + try { + Deno.kill(target, "SIGKILL"); + } catch { + // already exited + } + } +} + +export interface RunQuartoOptions { + // Per-test environment overlay. + env?: Record; + // Binary-mode working directory. + cwd?: string; + // Binary-mode log target and options. + logFile?: string; + logLevel?: string; + logFormat?: string; + timeoutMs?: number; + // Binary mode only. Defaults to true; testQuartoCmd disables it so + // verifiers receive failures through log records. + throwOnFailure?: boolean; +} + +export interface RunQuartoResult { + // Present only for binary-mode spawns. Dev mode has no exit code - it + // reports failure by rejecting instead (see runDevQuarto). + code?: number; + timedOut: boolean; + stderrTail?: string; +} + +// Dispatch to the in-process dev sources or the configured built binary. +export async function runQuarto( + args: string[], + options: RunQuartoOptions = {}, +): Promise { + const bin = quartoTestBin(); + return bin + ? runBinaryQuarto(bin, args, options) + : runDevQuarto(args, options); +} + +// A dev-mode timeout rejects but cannot stop the in-process render. +async function runDevQuarto( + args: string[], + options: RunQuartoOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + }); + try { + await Promise.race([quarto(args, undefined, options.env), timeout]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } + // quarto() either resolves or rejects: on CommandError or commandFailed() + // it calls exitWithCleanup(1), which Deno.exits the whole test process + // before this function could return a failure code anyway. + return { timedOut: false }; +} + +// Spawn the built binary and enforce timeout, logging, and failure policy. +async function runBinaryQuarto( + bin: string, + args: string[], + options: RunQuartoOptions, +): Promise { + assertTestBinary(bin); + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + const throwOnFailure = options.throwOnFailure ?? true; + + // LogFileHandler truncates its target, so each child writes a temporary + // log that is merged into the test log after exit. + const spawnArgs = [...args]; + let childLog: string | undefined; + if (options.logFile) { + childLog = Deno.makeTempFileSync({ suffix: ".json" }); + spawnArgs.push( + "--log", + childLog, + "--log-format", + options.logFormat ?? "json-stream", + // per-test log intent must land in the flags: explicit flags beat + // QUARTO_LOG_LEVEL env in quarto's logOptions, so passing the env + // var through would be silently ignored + "--log-level", + options.logLevel ?? options.env?.QUARTO_LOG_LEVEL ?? "info", + ); + } + + const child = new Deno.Command(bin, { + args: spawnArgs, + cwd: options.cwd ?? Deno.cwd(), + env: buildBinaryEnv(options.env), + clearEnv: true, + stdout: "piped", + stderr: "piped", + }).spawn(); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + // child.output() resolves after the kill; avoid an unhandled rejection. + killProcessTree(child.pid).catch(() => {}); + }, timeoutMs); + + // Drain both streams to avoid pipe-buffer deadlocks. + const output = await child.output(); + clearTimeout(timer); + + const stderrText = new TextDecoder().decode(output.stderr); + const stderrTail = stderrText.split("\n").slice(-25).join("\n").trim(); + const commandLine = `quarto ${args.join(" ")}`; + + if (options.logFile && childLog) { + mergeChildLog(options.logFile, childLog, { + timedOut, + code: output.code, + timeoutMs, + commandLine, + stderrTail, + }); + } + + if ((output.code !== 0 || timedOut) && throwOnFailure) { + throw new Error( + timedOut + ? `${commandLine} timed out after ${timeoutMs}ms` + : `${commandLine} exited with code ${output.code}\nstderr (tail):\n${stderrTail}`, + ); + } + + return { code: output.code, timedOut, stderrTail }; +} + +// Merge the child log and synthesize an ERROR when a failed child logged none. +function mergeChildLog( + logFile: string, + childLog: string, + outcome: { + timedOut: boolean; + code: number; + timeoutMs: number; + commandLine: string; + stderrTail: string; + }, +) { + let childContent = ""; + try { + childContent = Deno.readTextFileSync(childLog); + } catch { + // child never wrote the log (e.g. failed before logger init) + } + try { + Deno.removeSync(childLog); + } catch { + // best effort + } + // Only a timeout kill can tear a line; a clean exit flushes whole records. + if (outcome.timedOut) { + childContent = stripTornTrailingLine(childContent); + } + // A quiet successful command still needs an empty log for its verifiers. + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // log file may not exist yet + } + if (childContent.length > 0) { + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + childContent); + } else if (existing.length === 0) { + Deno.writeTextFileSync(logFile, ""); + } + if (outcome.timedOut) { + appendLogError( + logFile, + `${outcome.commandLine} timed out after ${outcome.timeoutMs}ms and was killed`, + ); + } else if (outcome.code !== 0 && !hasErrorRecordText(childContent)) { + // Startup and commandFailed paths can exit without logging an error. + appendLogError( + logFile, + `${outcome.commandLine} exited with code ${outcome.code} without logging an error\n` + + `stderr (tail):\n${outcome.stderrTail}`, + ); + } +} diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index f0f3ef77f7c..cf2c9d2d6a6 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -66,6 +66,52 @@ If ($null -eq $Env:QUARTO_DENO_DIR) { $Env:DENO_DIR = $Env:QUARTO_DENO_DIR } +# QUARTO_TEST_BIN selects an installed Quarto outside this checkout. +# The harness still uses the dev runtime configured above. +If (-not [string]::IsNullOrEmpty($Env:QUARTO_TEST_BIN)) { + If (-not (Test-Path $Env:QUARTO_TEST_BIN)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) does not exist" + Exit 1 + } + # Strip dev paths while probing the installed binary. + $probeStrip = @( + "QUARTO_SHARE_PATH", "QUARTO_BIN_PATH", "QUARTO_DEBUG", "DENO_DIR", + "QUARTO_DENO", "QUARTO_DENO_DOM", "QUARTO_ROOT", "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION" + ) + $probeSaved = @{} + ForEach ($name in $probeStrip) { + $probeSaved[$name] = [Environment]::GetEnvironmentVariable($name) + Remove-Item "Env:$name" -ErrorAction SilentlyContinue + } + Try { + $QUARTO_TEST_BIN_VERSION = & $Env:QUARTO_TEST_BIN --version + $QUARTO_TEST_BIN_PROBE_EXIT = $LASTEXITCODE + } Finally { + ForEach ($name in $probeStrip) { + If ($null -ne $probeSaved[$name]) { + [Environment]::SetEnvironmentVariable($name, $probeSaved[$name]) + } + } + } + If ($QUARTO_TEST_BIN_PROBE_EXIT -ne 0) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) exited with code $QUARTO_TEST_BIN_PROBE_EXIT while reporting its version." + Exit 1 + } + If ([string]::IsNullOrWhiteSpace($QUARTO_TEST_BIN_VERSION)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) did not report a version." + Write-Host -ForegroundColor red "The distribution is likely incomplete (missing share/version)." + Exit 1 + } + If ($QUARTO_TEST_BIN_VERSION -eq "99.9.9") { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + Write-Host -ForegroundColor red "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." + Write-Host -ForegroundColor red "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + Exit 1 + } + Write-Host "> BINARY MODE: testing built quarto $QUARTO_TEST_BIN_VERSION at $($Env:QUARTO_TEST_BIN)" +} + # Preparing running Deno with default arguments $QUARTO_IMPORT_MAP_ARG="--importmap=$(Join-Path $QUARTO_SRC_DIR "import_map.json")" @@ -161,6 +207,12 @@ If ($customArgs[0] -notlike "*smoke-all.test.ts") { $TESTS_TO_RUN=$customArgs } +# Binary mode defaults to smoke tests; other compatible suites are explicit. +If (-not [string]::IsNullOrEmpty($Env:QUARTO_TEST_BIN) -and $TESTS_TO_RUN.count -eq 0 -and $customArgs.count -eq 0) { + $TESTS_TO_RUN = @("smoke/") + Write-Host "> BINARY MODE: defaulting to smoke/ tests (pass a path explicitly to run others, e.g. integration/playwright-tests.test.ts)" +} + # ---- Running tests with Deno ------- $DENO_ARGS = @() diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a2c134fd392..8ea25e5dafd 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -62,6 +62,36 @@ export QUARTO_DEBUG=true QUARTO_DENO_OPTIONS="--config test-conf.json --v8-flags=--enable-experimental-regexp-engine,--max-old-space-size=8192,--max-heap-size=8192 --unstable-kv --unstable-ffi --no-lock --allow-all" +# QUARTO_TEST_BIN selects an installed Quarto outside this checkout. +# The harness still uses the dev runtime configured above. +if [[ -n "$QUARTO_TEST_BIN" ]]; then + if [[ ! -x "$QUARTO_TEST_BIN" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" + exit 1 + fi + # Strip dev paths while probing the installed binary. + QUARTO_TEST_BIN_VERSION="$(env -u QUARTO_SHARE_PATH -u QUARTO_BIN_PATH \ + -u QUARTO_DEBUG -u DENO_DIR -u QUARTO_DENO -u QUARTO_DENO_DOM \ + -u QUARTO_ROOT -u QUARTO_SRC_PATH -u QUARTO_FORCE_VERSION \ + "$QUARTO_TEST_BIN" --version 2>/dev/null)" + QUARTO_TEST_BIN_PROBE_EXIT=$? + if [[ $QUARTO_TEST_BIN_PROBE_EXIT -ne 0 ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) exited with code $QUARTO_TEST_BIN_PROBE_EXIT while reporting its version." + exit 1 + fi + if [[ -z "$QUARTO_TEST_BIN_VERSION" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) did not report a version." + echo "The distribution is likely incomplete (missing share/version)." + exit 1 + fi + if [[ "$QUARTO_TEST_BIN_VERSION" == "99.9.9" ]]; then + echo "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + echo "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." + echo "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + exit 1 + fi + echo "> BINARY MODE: testing built quarto ${QUARTO_TEST_BIN_VERSION} at ${QUARTO_TEST_BIN}" +fi if [[ -z $GITHUB_ACTION ]] && [[ -z $QUARTO_TESTS_NO_CONFIG ]] then @@ -161,6 +191,11 @@ else TESTS_TO_RUN=("${SMOKE_ALL_TEST_FILE}" "--" "${SMOKE_ALL_FILES[@]}") fi fi + # Binary mode defaults to smoke tests; other compatible suites are explicit. + if [[ -n "$QUARTO_TEST_BIN" && "${#TESTS_TO_RUN[@]}" -eq 0 && -z "$*" ]]; then + TESTS_TO_RUN=("smoke/") + echo "> BINARY MODE: defaulting to smoke/ tests (pass a path explicitly to run others, e.g. integration/playwright-tests.test.ts)" + fi # TESTS_TO_RUN is an array and quoted here on purpose: a bucket can be a # literal, unexpanded ** glob pattern (e.g. from the ff-matrix CI bucket), # and smoke-all.test.ts expands it itself via expandGlobSync. Expanding it diff --git a/tests/smoke/axe/axe-exit-codes.test.ts b/tests/smoke/axe/axe-exit-codes.test.ts index 5f98b4b05c4..ebadd870c0e 100644 --- a/tests/smoke/axe/axe-exit-codes.test.ts +++ b/tests/smoke/axe/axe-exit-codes.test.ts @@ -19,22 +19,11 @@ import { assert, assertEquals } from "testing/asserts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; -import { isWindows } from "../../../src/deno_ral/platform.ts"; import { execProcess } from "../../../src/core/process.ts"; import { unitTest } from "../../test.ts"; -import { quartoDevCmd } from "../../utils.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { AxeFindings } from "../../../src/command/call/axe/schemas.ts"; -// Under run-tests.sh/.ps1 the dev binary's dir is exported; resolve it -// explicitly so the subprocess never falls through to a release quarto that -// happens to be on PATH. -function quartoBin(): string { - const binPath = Deno.env.get("QUARTO_BIN_PATH"); - return binPath - ? join(binPath, isWindows ? "quarto.cmd" : "quarto") - : quartoDevCmd(); -} - const workingDir = Deno.makeTempDirSync({ prefix: "quarto-axe-exit" }); Deno.mkdirSync(join(workingDir, "site")); Deno.writeTextFileSync( @@ -61,10 +50,10 @@ function axeExitTest( ) { unitTest(name, async () => { const result = await execProcess({ - cmd: quartoBin(), + cmd: quartoDevBinCmd(), args: ["call", "axe", "site", ...args], cwd: workingDir, - env, + ...quartoSpawnEnvOptions(env), stdout: "piped", stderr: "piped", }); @@ -175,10 +164,10 @@ unitTest( Deno.writeTextFileSync(staleFindings, `{"stale": true}`); const result = await execProcess({ - cmd: quartoBin(), + cmd: quartoDevBinCmd(), args: ["call", "axe", "site"], cwd: workingDir, - env: { QUARTO_CHROMIUM: notABrowser }, + ...quartoSpawnEnvOptions({ QUARTO_CHROMIUM: notABrowser }), stdout: "piped", stderr: "piped", }); diff --git a/tests/smoke/axe/shared.ts b/tests/smoke/axe/shared.ts index 80d8ece0c51..0a82f159d14 100644 --- a/tests/smoke/axe/shared.ts +++ b/tests/smoke/axe/shared.ts @@ -16,7 +16,7 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { ExecuteOutput, testQuartoCmd, Verify } from "../../test.ts"; import { fileExists, validJsonWithFields } from "../../verify.ts"; import { docs } from "../../utils.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { AxeCell } from "../../../src/command/call/axe/scan.ts"; import { AxeFinding, @@ -158,7 +158,7 @@ export function axeSmokeTest( { cwd: () => siteDir, setup: async () => { - await quarto(["render"]); + await runQuarto(["render"]); }, teardown: () => { for (const dir of ["_site", "_axe-checks"]) { diff --git a/tests/smoke/convert/issue-12318.test.ts b/tests/smoke/convert/issue-12318.test.ts index 13d95b9e06c..88a37fcb1c0 100644 --- a/tests/smoke/convert/issue-12318.test.ts +++ b/tests/smoke/convert/issue-12318.test.ts @@ -11,32 +11,52 @@ import { test, } from "../../test.ts"; import { assert } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; (() => { const input = "docs/convert/issue-12318"; + const roundtrip = "issue-12318-2.qmd"; test({ // The name of the test name: "issue-12318", - + // Sets up the test context: { teardown: async () => { if (existsSync(input + '.ipynb')) { Deno.removeSync(input + '.ipynb'); } + if (existsSync(roundtrip)) { + Deno.removeSync(roundtrip); + } } }, - + // Executes the test - execute: async () => { - await quarto(["convert", "docs/convert/issue-12318.qmd"]); - await quarto(["convert", "docs/convert/issue-12318.ipynb", "--output", "issue-12318-2.qmd"]); - const txt = Deno.readTextFileSync("issue-12318-2.qmd"); - assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + execute: async (logFile?: string) => { + await runQuarto(["convert", input + ".qmd"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["convert", input + ".ipynb", "--output", roundtrip], { + logFile, + throwOnFailure: false, + }); }, - - verify: [], + + // Run assertions in verifiers so failures propagate through the harness. + verify: [ + noErrors, + { + name: "no triple backticks mid-line after roundtrip", + verify: (_outputs: ExecuteOutput[]) => { + const txt = Deno.readTextFileSync(roundtrip); + assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + return Promise.resolve(); + }, + }, + ], type: "unit" }); })(); diff --git a/tests/smoke/create/create.test.ts b/tests/smoke/create/create.test.ts index dc685b9d492..a699fbd50bb 100644 --- a/tests/smoke/create/create.test.ts +++ b/tests/smoke/create/create.test.ts @@ -11,6 +11,7 @@ import { walkSync } from "../../../src/deno_ral/fs.ts"; import { CreateResult } from "../../../src/command/create/cmd-types.ts"; import { assert } from "testing/asserts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; const kCreateTypes: Record = { "project": ["website", "default", "book", "website:blog"], @@ -54,6 +55,7 @@ for (const type of Object.keys(kCreateTypes)) { args: cmd.slice(1), stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }, stdIn); assert(process.success, process.stderr); if (process.stdout) { @@ -62,11 +64,11 @@ for (const type of Object.keys(kCreateTypes)) { assert(process.success, process.stderr); }); - // Verify all created files are user-writable. - // NOTE: In dev environments, resource files are already writable (0o644), - // so this test passes even without ensureUserWritable. It guards against - // regressions; the unit test in file-permissions.test.ts covers the - // read-only → writable transition directly. + // In dev environments resource files are already writable (0o644), so + // this passes even without ensureUserWritable; it guards against + // regressions. The unit test covers the read-only-to-writable + // transition directly. This smoke test checks the permissions of the + // created project. await t.step({ name: `> check writable ${type} ${template}`, ignore: Deno.build.os === "windows", @@ -101,6 +103,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(buildProcess.success, buildProcess.stderr); } @@ -115,6 +118,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(process.success, process.stderr); } diff --git a/tests/smoke/crossref/syntax.test.ts b/tests/smoke/crossref/syntax.test.ts index 8d4f913d718..5e1fc1f1983 100644 --- a/tests/smoke/crossref/syntax.test.ts +++ b/tests/smoke/crossref/syntax.test.ts @@ -4,7 +4,7 @@ * Copyright (C) 2020-2022 Posit Software, PBC */ -import { ensureFileRegexMatches } from "../../verify.ts"; +import { ensureFileRegexMatches, noErrors } from "../../verify.ts"; import { testRender } from "../render/render.ts"; import { crossref } from "./utils.ts"; import { @@ -15,7 +15,8 @@ import { Verify, } from "../../test.ts"; import { assert, fail } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; +import { safeRemoveSync } from "../../../src/deno_ral/fs.ts"; const syntaxQmd = crossref("syntax.qmd", "html"); testRender(syntaxQmd.input, "html", false, [ @@ -58,22 +59,30 @@ const verify: Verify = { }; const context: TestContext = { teardown: () => { - Deno.removeSync(imgQmd.output.outputPath); - Deno.removeSync(imgQmd.output.supportPath, { recursive: true }); + // Missing outputs must not let teardown mask the render failure. + safeRemoveSync(imgQmd.output.outputPath); + safeRemoveSync(imgQmd.output.supportPath, { recursive: true }); - Deno.removeSync(divQmd.output.outputPath); - Deno.removeSync(divQmd.output.supportPath, { recursive: true }); + safeRemoveSync(divQmd.output.outputPath); + safeRemoveSync(divQmd.output.supportPath, { recursive: true }); return Promise.resolve(); }, }; const testDesc: TestDescriptor = { // FIXME: why is this test flaky now? Ask @dragonstyle name: "test html produced by different figure syntax", context, - execute: async () => { - await quarto(["render", imgQmd.input]); - await quarto(["render", divQmd.input]); + execute: async (logFile?: string) => { + // Report render errors before comparing output files. + await runQuarto(["render", imgQmd.input], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", divQmd.input], { + logFile, + throwOnFailure: false, + }); }, - verify: [verify], + verify: [noErrors, verify], type: "smoke", }; test(testDesc); diff --git a/tests/smoke/engine/invalid-engine-in-project.test.ts b/tests/smoke/engine/invalid-engine-in-project.test.ts index 4b7081e94f8..782dbd764f5 100644 --- a/tests/smoke/engine/invalid-engine-in-project.test.ts +++ b/tests/smoke/engine/invalid-engine-in-project.test.ts @@ -1,19 +1,16 @@ -import { assertRejects } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; +import { testQuartoCmd } from "../../test.ts"; +import { printsMessage } from "../../verify.ts"; -test( - { - name: "invalid engines option errors", - execute: async () => { - assertRejects( - async () => {await quarto(["render", "docs/engine/invalid-project/notebook.qmd"])}, - Error, - "'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine", - ) - }, - type: "smoke", - context: {}, - verify: [], - } -) \ No newline at end of file +testQuartoCmd( + "render", + ["docs/engine/invalid-project/notebook.qmd"], + [ + printsMessage({ + level: "ERROR", + regex: + /'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine/, + }), + ], + {}, + "invalid engines option errors", +); diff --git a/tests/smoke/env/check.test.ts b/tests/smoke/env/check.test.ts index 9125b71d9c9..d0b41d09899 100644 --- a/tests/smoke/env/check.test.ts +++ b/tests/smoke/env/check.test.ts @@ -5,13 +5,20 @@ * */ import { testQuartoCmd } from "../../test.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { noErrorsOrWarnings, printsMessage } from "../../verify.ts"; +// Dev mode reports the 99.9.9 sentinel version; a built binary reports its +// real version, so only require a semver-shaped version line there. +const versionRegex = isBinaryMode() + ? /Version: \d+\.\d+\.\d+/ + : /Version: 99\.9\.9/; + testQuartoCmd( "check", [], [ noErrorsOrWarnings, - printsMessage({level: "INFO", regex: /Version: 99\.9\.9/}), + printsMessage({level: "INFO", regex: versionRegex}), ], ); diff --git a/tests/smoke/extensions/extension-render-journals.test.ts b/tests/smoke/extensions/extension-render-journals.test.ts index 0327048fb64..9137c9f9819 100644 --- a/tests/smoke/extensions/extension-render-journals.test.ts +++ b/tests/smoke/extensions/extension-render-journals.test.ts @@ -5,10 +5,11 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const journalRepos = [ // { repo: "acm", noSupporting: true }, TODO this format needs changes after this merge. @@ -42,15 +43,14 @@ for (const journalRepo of journalRepos) { // Sets up the test setup: async () => { console.log(`using quarto-journals/${journalRepo.repo}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await quarto([ - "use", - "template", - `quarto-journals/${journalRepo.repo}`, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + `quarto-journals/${journalRepo.repo}`, + "--no-prompt", + ]); + }); }, // Cleans up the test diff --git a/tests/smoke/extensions/extension-render-typst-templates.test.ts b/tests/smoke/extensions/extension-render-typst-templates.test.ts index 5acdbe91487..1d62194c5b7 100644 --- a/tests/smoke/extensions/extension-render-typst-templates.test.ts +++ b/tests/smoke/extensions/extension-render-typst-templates.test.ts @@ -5,10 +5,11 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const GITHUB_REPO = "quarto-ext/typst-templates"; @@ -40,15 +41,14 @@ for (const name of typstTemplates) { setup: async () => { const source = `${GITHUB_REPO}/${name}`; console.log(`using template: ${source}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await quarto([ - "use", - "template", - source, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + source, + "--no-prompt", + ]); + }); }, teardown: async () => { diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 77729c882e1..dfb601f128b 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -5,20 +5,18 @@ */ import { docs } from "../../utils.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { assertEquals } from "testing/asserts"; -import { isWindows } from "../../../src/deno_ral/platform.ts"; async function runEditorSupportCrossref(doc: string) { - const cmdLine: string = isWindows ? - "../package/dist/bin/quarto.cmd" : - "../package/dist/bin/quarto"; - - const cmd = new Deno.Command(cmdLine, { + // Use the built test binary in binary mode; otherwise pin the local CLI. + const cmd = new Deno.Command(quartoDevBinCmd(), { args: ["editor-support", "crossref"], stdin: "piped", stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); const child = cmd.spawn(); const writer = child.stdin.getWriter(); @@ -26,7 +24,7 @@ async function runEditorSupportCrossref(doc: string) { Deno.readTextFileSync(doc), ); await writer.write(buf); - writer.releaseLock(); + // close() sends EOF; releaseLock() would detach the writer before close(). await writer.close(); const outputBuf = await child.output(); const status = await child.status; @@ -36,15 +34,21 @@ async function runEditorSupportCrossref(doc: string) { return json; } +// Run assertions in verifiers so failures propagate through the harness. test({ name: "editor-support:crossref:smoke-1", context: {}, - execute: async () => { - const json = await runEditorSupportCrossref(docs("crossrefs/sections.qmd")); - assertEquals(json.entries[0].key, "sec-introduction"); - assertEquals(json.entries[0].caption, "Introduction"); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref output", + verify: async (_outputs) => { + const json = await runEditorSupportCrossref( + docs("crossrefs/sections.qmd"), + ); + assertEquals(json.entries[0].key, "sec-introduction"); + assertEquals(json.entries[0].caption, "Introduction"); + }, + }], type: "smoke", }); @@ -52,10 +56,13 @@ function smokeTestCrossref(name: string, doc: string) { test({ name, context: {}, - execute: async () => { - await runEditorSupportCrossref(doc); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref runs cleanly", + verify: async (_outputs) => { + await runEditorSupportCrossref(doc); + }, + }], type: "smoke", }); } diff --git a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts index 2f26608179b..b47758665ee 100644 --- a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts +++ b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts @@ -7,15 +7,15 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { _setIsRStudioForTest } from "../../../src/core/platform.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { ExecuteOutput, testQuartoCmd, } from "../../test.ts"; import { assert, assertEquals } from "testing/asserts"; -// Test: standalone file inspect with RStudio override should NOT emit project. -// Uses _setIsRStudioForTest to avoid Deno.env.set() race conditions in -// parallel tests (see #14218, PR #12621). +// Dev mode uses the test hook to avoid process-global environment races. +// Binary mode passes RSTUDIO=1 to the spawned Quarto. (() => { const input = "docs/inspect/standalone-hello.qmd"; const output = "docs/inspect/standalone-hello.json"; @@ -34,11 +34,16 @@ import { assert, assertEquals } from "testing/asserts"; } ], { + env: isBinaryMode() ? { RSTUDIO: "1" } : undefined, setup: async () => { - _setIsRStudioForTest(true); + if (!isBinaryMode()) { + _setIsRStudioForTest(true); + } }, teardown: async () => { - _setIsRStudioForTest(undefined); + if (!isBinaryMode()) { + _setIsRStudioForTest(undefined); + } if (existsSync(output)) { Deno.removeSync(output); } diff --git a/tests/smoke/issues/9133/9133.test.ts b/tests/smoke/issues/9133/9133.test.ts index d0b21db04a0..c1df612c02b 100644 --- a/tests/smoke/issues/9133/9133.test.ts +++ b/tests/smoke/issues/9133/9133.test.ts @@ -1,23 +1,19 @@ -import { quarto } from "../../../../src/quarto.ts"; +import { runQuarto } from "../../../quarto-cmd.ts"; import { test } from "../../../test.ts"; if (Deno.build.os !== "windows") { test({ - name: "https://github.com/quarto-dev/quarto-cli/issues/9133", + name: "https://github.com/quarto-dev/quarto-cli/issues/9133", context: { setup: async () => { Deno.mkdirSync("smoke/issues/9133/oh'\"no", { recursive: true }); Deno.copyFileSync("smoke/issues/9133/jl", "smoke/issues/9133/oh'\"no/jl.qmd"); Deno.copyFileSync("smoke/issues/9133/py", "smoke/issues/9133/oh'\"no/py.qmd"); - const timeout = new Promise((_resolve, reject) => { - setTimeout(reject, 600000, "timed out after 10 minutes"); - }); - await Promise.race([ - Promise.all([ - quarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), - quarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), - ]), - timeout, + // Binary mode uses separate processes, so the in-process race may not + // reproduce there. Avoid a shared log for concurrent children. + await Promise.all([ + runQuarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), + runQuarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), ]); } }, diff --git a/tests/smoke/jupyter/cache.test.ts b/tests/smoke/jupyter/cache.test.ts index 6a687778aab..c2f2b006247 100644 --- a/tests/smoke/jupyter/cache.test.ts +++ b/tests/smoke/jupyter/cache.test.ts @@ -4,10 +4,10 @@ * Copyright (C) 2023 Posit Software, PBC */ import { dirname, join } from "path"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { docs } from "../../utils.ts"; -import { folderExists, printsMessage } from "../../verify.ts"; +import { folderExists, noErrors, printsMessage } from "../../verify.ts"; import { fileLoader } from "../../utils.ts"; import { safeExistsSync, safeRemoveSync } from "../../../src/core/path.ts"; @@ -16,14 +16,17 @@ const cacheFolder = join(dirname(testInput.input), ".jupyter_cache") test({ name: "Jupyter cache is working", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); + // Render twice to exercise the cache. + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -39,6 +42,7 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder), // this will check only for the second render that should be read from cache printsMessage({ level: "INFO", regex: /Notebook read from cache/}) @@ -54,14 +58,17 @@ const cacheFolder2 = join(dirname(testInput2.input), ".cache/jupyter-cache") test({ name: "Jupyter cache folder can be change", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); + // Render twice to exercise the cache. + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -80,9 +87,10 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder2), // this will check only for the second render that should be read from cache printsMessage({level: "INFO", regex: /Notebook read from cache/}) ], type: "smoke", -}); \ No newline at end of file +}); diff --git a/tests/smoke/jupyter/issue-10097.test.ts b/tests/smoke/jupyter/issue-10097.test.ts index 19586200e28..743e237a76f 100644 --- a/tests/smoke/jupyter/issue-10097.test.ts +++ b/tests/smoke/jupyter/issue-10097.test.ts @@ -1,26 +1,25 @@ /* * parameter-label-duplication.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/10097 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:parameter:label-duplication", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/10097 - await quarto(["render", - "docs/jupyter/parameters/issue-10097.qmd", - "--execute-param", 'datapath:"weird"', - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/10097 +testQuartoCmd( + "render", + [ + "docs/jupyter/parameters/issue-10097.qmd", + "--execute-param", + 'datapath:"weird"', + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:parameter:label-duplication", +); diff --git a/tests/smoke/jupyter/issue-12374.test.ts b/tests/smoke/jupyter/issue-12374.test.ts index 56c5ce67dc0..3007adb3499 100644 --- a/tests/smoke/jupyter/issue-12374.test.ts +++ b/tests/smoke/jupyter/issue-12374.test.ts @@ -1,25 +1,23 @@ /* * issue-12374.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/12374 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:issue-12374.test.ts", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/12374 - await quarto(["render", - "docs/jupyter/issue-12374.ipynb", - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/12374 +testQuartoCmd( + "render", + [ + "docs/jupyter/issue-12374.ipynb", + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:issue-12374.test.ts", +); diff --git a/tests/smoke/logging/log-level-and-formats.test.ts b/tests/smoke/logging/log-level-and-formats.test.ts index 9c2bbf703d0..68957613228 100644 --- a/tests/smoke/logging/log-level-and-formats.test.ts +++ b/tests/smoke/logging/log-level-and-formats.test.ts @@ -10,6 +10,7 @@ import { execProcess } from "../../../src/core/process.ts"; import { md5HashSync } from "../../../src/core/hash.ts"; import { safeRemoveIfExists } from "../../../src/core/path.ts"; import { quartoDevCmd, outputForInput } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { assert } from "testing/asserts"; import { LogFormat } from "../../../src/core/log.ts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; @@ -90,7 +91,8 @@ function testLogDirectly(options: { cmd: quartoDevCmd(), args: args, stdout: "piped", - stderr: "piped" + stderr: "piped", + ...quartoSpawnEnvOptions(), }); // Get stdout/stderr with fallback to empty string @@ -137,35 +139,31 @@ function testLogDirectly(options: { } - // If JSON format is specified, verify the output is valid JSON + // Catch parsing errors only; assertion failures must propagate. if (logFile && options.format === "json-stream") { assert(existsSync(logFile), "Log file should exist"); - let foundValidJson = false; + let outputs; try { - const outputs = readExecuteOutput(logFile); - foundValidJson = true; - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldNotContainLevel?.includes(out.levelName)).forEach( - (out) => { - assert(false, `JSON output should not contain level ${out.levelName}, but found: ${out.msg}`); - } + outputs = readExecuteOutput(logFile); + } catch { + outputs = undefined; + } + assert(outputs !== undefined, "JSON format should produce valid JSON output"); + const records = outputs!.filter((out) => out.msg !== ""); + const levels = new Set(records.map((out) => out.levelName)); + for (const lvl of options.expectedOutputs?.shouldNotContainLevel ?? []) { + const offending = records.find((out) => out.levelName === lvl); + assert( + offending === undefined, + `JSON log should not contain level ${lvl}, but found: ${offending?.msg}` ); - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldContainLevel?.includes(out.levelName)).forEach( - (out) => { - let json = undefined; - try { - json = JSON.parse(out.msg); - } catch { - assert(false, "Error parsing JSON returned by quarto meta"); - } - assert( - Object.keys(json).length > 0, - "JSON returned by quarto meta seems invalid", - ); - } + } + for (const lvl of options.expectedOutputs?.shouldContainLevel ?? []) { + assert( + levels.has(lvl), + `JSON log should contain at least one ${lvl} record; found levels: ${[...levels].join(", ") || "(none)"}` ); - - } catch (e) {} - assert(foundValidJson, "JSON format should produce valid JSON output"); + } } } finally { // Clean up log file if it exists diff --git a/tests/smoke/lua-unit/lua-unit.test.ts b/tests/smoke/lua-unit/lua-unit.test.ts index 5a4f61341b9..7715146d5ae 100644 --- a/tests/smoke/lua-unit/lua-unit.test.ts +++ b/tests/smoke/lua-unit/lua-unit.test.ts @@ -21,6 +21,7 @@ import { fromFileUrl, join } from "../../../src/deno_ral/path.ts"; import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; // Explicit list, relative to tests/unit-lua/. Keep alphabetized. @@ -52,7 +53,7 @@ for (const relPath of LUA_TESTS) { { cmd: quartoDevCmd(), args: ["run", luaScript], - env: { LUA_PATH }, + ...quartoSpawnEnvOptions({ LUA_PATH }), }, undefined, undefined, diff --git a/tests/smoke/project/project-prepost.test.ts b/tests/smoke/project/project-prepost.test.ts index b7895f4a5f1..5f827d336d9 100644 --- a/tests/smoke/project/project-prepost.test.ts +++ b/tests/smoke/project/project-prepost.test.ts @@ -63,18 +63,22 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/extension")], - [{ - name: "i-exist.txt exists", + // Verify success before checking the negative path condition. + [noErrors, { + name: "prepost extension file effects", verify: async () => { - const path = join(docs("project/prepost/extension"), "i-exist.txt"); - verifyNoPath(path); + verifyNoPath(join(docs("project/prepost/extension"), "i-exist.txt")); + verifyPath(join(docs("project/prepost/extension"), "i-was-created.txt")); } }], { + // Remove stale artifacts so verification covers this render. + setup: async () => { + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-exist.txt")); + }, teardown: async () => { - const path = join(docs("project/prepost/extension"), "i-was-created.txt"); - verifyPath(path); - safeRemoveIfExists(path); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); const siteDir = join(docs("project/prepost/extension"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); @@ -85,19 +89,26 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/issue-10828")], - [], + [noErrors, { + name: "project input/output files written", + verify: async () => { + verifyPath(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + verifyPath(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + } + }], { env: { "QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES": normalizePath(docs("project/prepost/issue-10828/input-files.txt")), "QUARTO_USE_FILE_FOR_PROJECT_OUTPUT_FILES": normalizePath(docs("project/prepost/issue-10828/output-files.txt")) }, + // Remove stale artifacts so verification covers this render. + setup: async () => { + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + }, teardown: async () => { - const inputPath = normalizePath(docs("project/prepost/issue-10828/input-files.txt")); - const outputPath = normalizePath(docs("project/prepost/issue-10828/output-files.txt")); - verifyPath(inputPath); - safeRemoveIfExists(inputPath); - verifyPath(outputPath); - safeRemoveIfExists(outputPath); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); const siteDir = join(docs("project/prepost/issue-10828"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); diff --git a/tests/smoke/project/project-stdout.test.ts b/tests/smoke/project/project-stdout.test.ts index dfa38d86299..b0539af5bb0 100644 --- a/tests/smoke/project/project-stdout.test.ts +++ b/tests/smoke/project/project-stdout.test.ts @@ -14,6 +14,7 @@ import { docs } from "../../utils.ts"; import { directoryEmptyButFor, fileExists, + noErrors, verifyYamlFile, } from "../../verify.ts"; @@ -30,7 +31,7 @@ const siteOutDir = join(siteProjDir, outDir); testQuartoCmd( "render", [siteProjDir, "-o", "-"], - [], + [noErrors], { teardown: async () => { if (existsSync(siteOutDir)) { diff --git a/tests/smoke/render/render-format-extension.test.ts b/tests/smoke/render/render-format-extension.test.ts index f82374cf32f..09eaa5ab758 100644 --- a/tests/smoke/render/render-format-extension.test.ts +++ b/tests/smoke/render/render-format-extension.test.ts @@ -12,8 +12,8 @@ // Both files serve different purposes and should remain separate. import { safeRemoveSync } from "../../../src/core/path.ts"; -import { docs } from "../../utils.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { docs, withCwd } from "../../utils.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { testRender } from "./render.ts"; @@ -28,19 +28,16 @@ import { testRender } from "./render.ts"; const updateExtensions = async () => { try { console.log("Updating quarto-journals extensions to latest versions..."); - const wd = Deno.cwd(); - Deno.chdir(docs("extensions/format/academic")); - - for (const repo of ["acs", "elsevier"]) { - await quarto([ - "update", - "extension", - `quarto-journals/${repo}`, - "--no-prompt", - ]); - } - - Deno.chdir(wd); + await withCwd(docs("extensions/format/academic"), async () => { + for (const repo of ["acs", "elsevier"]) { + await runQuarto([ + "update", + "extension", + `quarto-journals/${repo}`, + "--no-prompt", + ]); + } + }); console.log("Extensions updated successfully"); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/tests/smoke/render/render-freeze.test.ts b/tests/smoke/render/render-freeze.test.ts index 7291e24d798..b74a6d846b0 100644 --- a/tests/smoke/render/render-freeze.test.ts +++ b/tests/smoke/render/render-freeze.test.ts @@ -9,10 +9,10 @@ import { dirname, join } from "../../../src/deno_ral/path.ts"; import { assert } from "testing/asserts"; import { Metadata } from "../../../src/config/types.ts"; -import { removeIfEmptyDir } from "../../../src/core/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { removeIfEmptyDir, safeRemoveIfExists } from "../../../src/core/path.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ExecuteOutput, Verify } from "../../test.ts"; -import { outputCreated } from "../../verify.ts"; +import { noErrors, outputCreated } from "../../verify.ts"; import { testRender } from "./render.ts"; const regex = /output file: .*\.knit\.md/m; @@ -67,9 +67,8 @@ const ignoreFrozen = { const projectOutputExists: Verify = { name: "Make sure project output exists", - verify: (_output: ExecuteOutput[]) => { - outputCreated(path, "html"); - return Promise.resolve(); + verify: (output: ExecuteOutput[]) => { + return outputCreated(path, "html").verify(output); }, }; @@ -107,16 +106,16 @@ function testFileContext( markdown, ); - await quarto(["render", path]); + await runQuarto(["render", path]); }, teardown: async () => { // Clean up the test file await Deno.remove(path); await Deno.remove(quartoProj); - // Get rid of the freezer + // Get rid of the freezer (a failed render never creates it) const freezerDir = join(dirname(path), "_freeze"); - Deno.removeSync(join(freezerDir, testFileName), { recursive: true }); + safeRemoveIfExists(join(freezerDir, testFileName)); // Maybe clean up empty freeze dir removeIfEmptyDir(freezerDir); @@ -130,7 +129,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "clean fzr - auto", ...testContext, @@ -142,7 +141,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, ignoreFrozen], + [noErrors, projectOutputExists, ignoreFrozen], { name: "dirty fzr - auto", setup: async () => { @@ -167,7 +166,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "dirty fzr - freeze", setup: async () => { diff --git a/tests/smoke/render/render-output-file-collision.test.ts b/tests/smoke/render/render-output-file-collision.test.ts index f360ed0d54e..0b802f536f8 100644 --- a/tests/smoke/render/render-output-file-collision.test.ts +++ b/tests/smoke/render/render-output-file-collision.test.ts @@ -14,7 +14,7 @@ */ import { existsSync, safeRemoveSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { docs } from "../../utils.ts"; import { testQuartoCmd } from "../../test.ts"; import { @@ -103,7 +103,7 @@ testQuartoCmd( { setup: async () => { await cleanup(defaultDir, [...defaultOutputs, ".quarto"])(); - await quarto(["render", defaultDir]); + await runQuarto(["render", defaultDir]); }, teardown: cleanup(defaultDir, [...defaultOutputs, ".quarto"]), }, diff --git a/tests/smoke/run/command-passthrough.test.ts b/tests/smoke/run/command-passthrough.test.ts index 6db94e984e9..462580d896c 100644 --- a/tests/smoke/run/command-passthrough.test.ts +++ b/tests/smoke/run/command-passthrough.test.ts @@ -1,16 +1,18 @@ import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; const testPassthroughCmd = (name: string, command: string, args: string[]) => { unitTest(name, async () => { const result = await execProcess({ - cmd: quartoDevCmd(), + cmd: quartoDevCmd(), args: [ command, ...args, - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }); diff --git a/tests/smoke/run/run-script.test.ts b/tests/smoke/run/run-script.test.ts index ee6fb0f16fe..308a465ffce 100644 --- a/tests/smoke/run/run-script.test.ts +++ b/tests/smoke/run/run-script.test.ts @@ -3,6 +3,7 @@ import { ensureDirSync } from "../../../src/deno_ral/fs.ts"; import { assert, assertEquals } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; import { EOL } from "fs/eol"; import { lines } from "../../../src/core/text.ts"; @@ -20,9 +21,9 @@ const ensureStreams = (name: string, script: string, stdout: string, stderr: str basename(script), ], // disable logging here to allow for checking the output - env: { + ...quartoSpawnEnvOptions({ "QUARTO_LOG_LEVEL": "CRITICAL", - } + }), }, undefined, undefined, @@ -55,7 +56,8 @@ const testRunCmd = (name: string, script: string) => { args: [ "run", basename(script), - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }, diff --git a/tests/smoke/run/stdlib-run-version.test.ts b/tests/smoke/run/stdlib-run-version.test.ts index bbe7dadf726..eef54b05648 100644 --- a/tests/smoke/run/stdlib-run-version.test.ts +++ b/tests/smoke/run/stdlib-run-version.test.ts @@ -8,15 +8,18 @@ import { execProcess } from "../../../src/core/process.ts"; import { assert } from "testing/asserts"; import { unitTest } from "../../test.ts"; +import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { isWindows } from "../../../src/deno_ral/platform.ts"; unitTest("stdlib-run-version", async () => { const result = await execProcess({ - cmd: "quarto", + cmd: quartoDevCmd(), args: [ "run", "docs/run/test-stdlib.ts", ], + ...quartoSpawnEnvOptions(), }); console.log({result}) assert(result.success); diff --git a/tests/smoke/self-contained/stdout.test.ts b/tests/smoke/self-contained/stdout.test.ts index f03a8e6c730..5254c0a397a 100644 --- a/tests/smoke/self-contained/stdout.test.ts +++ b/tests/smoke/self-contained/stdout.test.ts @@ -1,11 +1,13 @@ -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; test({ name: "https://github.com/quarto-dev/quarto-cli/issues/11068", context: { setup: async() => { - await quarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); + // only asserts the render to stdout succeeds (failure throws); in + // binary mode the child's stdout is drained and discarded + await runQuarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); } }, execute: async () => {}, diff --git a/tests/smoke/site/render-site-themes.test.ts b/tests/smoke/site/render-site-themes.test.ts index e12fa17c25a..3bdef320f09 100644 --- a/tests/smoke/site/render-site-themes.test.ts +++ b/tests/smoke/site/render-site-themes.test.ts @@ -5,7 +5,7 @@ * */ -import { copySync } from "../../../src/deno_ral/fs.ts"; +import { copySync, safeRemoveSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { testQuartoCmd, Verify } from "../../test.ts"; import { docs } from "../../utils.ts"; @@ -75,8 +75,8 @@ try { return Promise.resolve(); }, teardown: () => { - // clean output - Deno.removeSync(path, { recursive: true }); + // clean output (setup may have failed before copying the fixture) + safeRemoveSync(path, { recursive: true }); return Promise.resolve(); }, }, diff --git a/tests/smoke/smoke-all.test.ts b/tests/smoke/smoke-all.test.ts index 32b0c69d14f..311412de449 100644 --- a/tests/smoke/smoke-all.test.ts +++ b/tests/smoke/smoke-all.test.ts @@ -57,7 +57,7 @@ import { findProjectDir, findProjectOutputDir, outputForInput } from "../utils.t import { jupyterNotebookToMarkdown } from "../../src/command/convert/jupyter.ts"; import { basename, dirname, join, relative } from "../../src/deno_ral/path.ts"; import { WalkEntry } from "../../src/deno_ral/fs.ts"; -import { quarto } from "../../src/quarto.ts"; +import { runQuarto } from "../quarto-cmd.ts"; import { safeExistsSync, safeRemoveSync } from "../../src/core/path.ts"; import { runningInCI } from "../../src/core/ci-info.ts"; @@ -434,7 +434,9 @@ for (const { path: fileName } of files) { projectPath && !renderedProjects.has(projectPath) ) { - await quarto(["render", projectPath]); + // fail-loudly pre-render (throwOnFailure defaults to true); + // dispatches to the built binary when QUARTO_TEST_BIN is set + await runQuarto(["render", projectPath]); renderedProjects.add(projectPath); } diff --git a/tests/smoke/typst-gather/typst-gather.test.ts b/tests/smoke/typst-gather/typst-gather.test.ts index 7920a63158c..94843aa8e2a 100644 --- a/tests/smoke/typst-gather/typst-gather.test.ts +++ b/tests/smoke/typst-gather/typst-gather.test.ts @@ -4,6 +4,16 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { execProcess } from "../../../src/core/process.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; + +// Remove persistent, gitignored caches before verifying generated content. +const freshCache = (cacheDir: string) => async () => { + if (existsSync(cacheDir)) { + Deno.removeSync(cacheDir, { recursive: true }); + } +}; + // Test 1: Auto-detection from _extension.yml const verifyPackagesCreated: Verify = { name: "Verify typst/packages directory was created", @@ -38,9 +48,10 @@ const verifyExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyPackagesCreated, verifyExamplePackageCached], + [noErrors, verifyPackagesCreated, verifyExamplePackageCached], { cwd: () => "smoke/typst-gather", + setup: freshCache("_extensions/test-format/typst"), }, "typst-gather caches preview packages from extension templates", ); @@ -78,9 +89,10 @@ const verifyConfigExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyConfigPackagesCreated, verifyConfigExamplePackageCached], + [noErrors, verifyConfigPackagesCreated, verifyConfigExamplePackageCached], { cwd: () => "smoke/typst-gather/with-config", + setup: freshCache("_extensions/config-format/typst"), }, "typst-gather uses rootdir from config file", ); @@ -246,7 +258,8 @@ const verifyNoPackagesStaged: Verify = { testQuartoCmd( "render", [join(noPackagesProjectDir, "index.qmd"), "--to", "typst"], - [verifyNoPackagesStaged], + // Require a successful render before checking that nothing was staged. + [noErrors, verifyNoPackagesStaged], { teardown: async () => { try { @@ -263,25 +276,20 @@ testQuartoCmd( ); // Helper to run quarto as an external process and capture exit code -async function runQuarto( +async function execTypstGather( args: string[], cwd: string, env?: Record, ): Promise<{ success: boolean; stdout: string; stderr: string }> { - const quartoCmd = Deno.build.os === "windows" ? "quarto.cmd" : "quarto"; - const quartoPath = join( - Deno.cwd(), - "..", - "package/dist/bin", - quartoCmd, - ); const result = await execProcess({ - cmd: quartoPath, + // Use the built test binary in binary mode; otherwise pin the local CLI. + cmd: quartoDevBinCmd(), args, cwd, stdout: "piped", stderr: "piped", - env: env ? { ...Deno.env.toObject(), ...env } : undefined, + // Binary mode strips dev-tree variables before applying the overlay. + ...quartoSpawnEnvOptions(env), }); return { success: result.success, @@ -299,7 +307,7 @@ unitTest( const configPath = join(cwd, "typst-gather.toml"); try { Deno.writeTextFileSync(configPath, "# existing config\n"); - const result = await runQuarto( + const result = await execTypstGather( ["call", "typst-gather", "--init-config"], cwd, ); @@ -319,7 +327,7 @@ unitTest( "typst-gather --init-config errors with no extension directory", async () => { const cwd = join(Deno.cwd(), "smoke/typst-gather/no-extension"); - const result = await runQuarto( + const result = await execTypstGather( ["call", "typst-gather", "--init-config"], cwd, ); @@ -338,7 +346,7 @@ unitTest( "typst-gather --init-config warns with empty extension (no typst entries)", async () => { const cwd = join(Deno.cwd(), "smoke/typst-gather/empty-extension"); - const result = await runQuarto( + const result = await execTypstGather( ["call", "typst-gather", "--init-config"], cwd, ); @@ -436,7 +444,7 @@ unitTest( Deno.cwd(), "docs/smoke-all/typst/marginalia-only-project", ); - const result = await runQuarto( + const result = await execTypstGather( ["render", "index.qmd", "--to", "typst"], projectDir, { QUARTO_TYPST_GATHER: "/nonexistent/typst-gather-binary" }, @@ -468,7 +476,7 @@ unitTest( Deno.cwd(), "docs/smoke-all/typst/marginalia-only-project", ); - const result = await runQuarto( + const result = await execTypstGather( ["render", "index.qmd", "--to", "typst"], projectDir, { QUARTO_TYPST_GATHER: falseCmd }, diff --git a/tests/smoke/verify/pdf-metadata.test.ts b/tests/smoke/verify/pdf-metadata.test.ts index 31ad2c775cb..5faa4675a16 100644 --- a/tests/smoke/verify/pdf-metadata.test.ts +++ b/tests/smoke/verify/pdf-metadata.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfMetadata } from "../../verify-pdf-metadata.ts"; import { assert } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,14 +42,23 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +// Remove stale output in setup and run assertions before teardown cleanup. +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf metadata assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/verify/pdf-text-position.test.ts b/tests/smoke/verify/pdf-text-position.test.ts index 2f40fa0e26d..5ec615f2fa5 100644 --- a/tests/smoke/verify/pdf-text-position.test.ts +++ b/tests/smoke/verify/pdf-text-position.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfTextPositions, PdfTextPositionAssertion } from "../../verify-pdf-text-position.ts"; import { assert, AssertionError } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,21 +42,30 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +// Remove stale output in setup and run assertions before teardown cleanup. +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf text position assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + await runSemanticTagTests(); + await runPageRoleTests(); + await runEdgeOverrideTests(); + await runDistanceConstraintTests(); + await runDistanceConstraintErrorTests(); + await runPageRoleWithEdgeTests(); + await runCenterEdgeTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - await runSemanticTagTests(); - await runPageRoleTests(); - await runEdgeOverrideTests(); - await runDistanceConstraintTests(); - await runDistanceConstraintErrorTests(); - await runPageRoleWithEdgeTests(); - await runCenterEdgeTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/website/drafts-env.test.ts b/tests/smoke/website/drafts-env.test.ts index 7e7584d4d8c..7a8cef50ac5 100644 --- a/tests/smoke/website/drafts-env.test.ts +++ b/tests/smoke/website/drafts-env.test.ts @@ -16,6 +16,8 @@ const renderDir = docs("websites/drafts/drafts-env"); const dir = join(Deno.cwd(), renderDir); const outDir = join(dir, "_site"); +// Dev mode caches the base profile on first render, so set it before tests. +// Binary mode receives the same value through context.env. Deno.env.set("QUARTO_PROFILE", "drafts"); testQuartoCmd( @@ -23,6 +25,7 @@ testQuartoCmd( [renderDir], [noErrorsOrWarnings, ...[doesntHaveContentLinksToDrafts, doesntHaveEnvelopeLinksToDrafts, draftPostIsEmpty, searchDoesntHaveDraft, siteMapDoesntHaveDraft].map((ver) => { return ver(outDir)})], { + env: { QUARTO_PROFILE: "drafts" }, teardown: async () => { if (existsSync(outDir)) { await Deno.remove(outDir, { recursive: true }); diff --git a/tests/test.ts b/tests/test.ts index 729568e41f1..4a182b54026 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -10,7 +10,7 @@ import { warning } from "../src/deno_ral/log.ts"; import { initDenoDom } from "../src/core/deno-dom.ts"; import { cleanupLogger, initializeLogger, flushLoggers, logError, LogLevel, LogFormat } from "../src/core/log.ts"; -import { quarto } from "../src/quarto.ts"; +import { appendLogError, isBinaryMode, runQuarto } from "./quarto-cmd.ts"; import { join } from "../src/deno_ral/path.ts"; import * as colors from "fmt/colors"; import { runningInCI } from "../src/core/ci-info.ts"; @@ -36,8 +36,8 @@ export interface TestDescriptor { // Sets up the test context: TestContext; - // Executes the test - execute: () => Promise; + // Binary mode passes the child log target. + execute: (logFile?: string) => Promise; // Used to verify the outcome of the test verify: Verify[]; @@ -90,6 +90,9 @@ export interface TestContext { // Defaults to 600000 (10 minutes). Lower it to assert a performance budget // (e.g. a render that must not regress into a hang). timeout?: number; + + // Ignore this test in binary mode because it requires in-process internals. + requiresDevQuarto?: boolean; } // Allow to merge test contexts in Tests helpers @@ -127,6 +130,8 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?: }, // override ignore if provided ignore: additionalContext.ignore ?? baseContext.ignore, + requiresDevQuarto: additionalContext.requiresDevQuarto ?? + baseContext.requiresDevQuarto, // merge env with additional context taking precedence env: { ...baseContext.env, ...additionalContext.env }, // override timeout if provided @@ -147,19 +152,16 @@ export function testQuartoCmd( } test({ name, - execute: async () => { - const timeoutMs = context?.timeout ?? 600000; - const timeout = new Promise((_resolve, reject) => { - setTimeout( - reject, - timeoutMs, - `timed out after ${timeoutMs}ms`, - ); + execute: async (logFile?: string) => { + await runQuarto([cmd, ...args], { + env: context?.env, + logFile, + logLevel: logConfig?.level, + logFormat: logConfig?.format, + timeoutMs: context?.timeout, + // Let verifiers report failures from the log. + throwOnFailure: false, }); - await Promise.race([ - quarto([cmd, ...args], undefined, context?.env), - timeout, - ]); }, verify, context: context || {}, @@ -213,7 +215,9 @@ export function test(test: TestDescriptor) { const sanitizeResources = test.context.sanitize?.resources; const sanitizeOps = test.context.sanitize?.ops; const sanitizeExit = test.context.sanitize?.exit; - const ignore = test.context.ignore; + // dev-only tests are ignored when targeting an external built binary + const ignore = test.context.ignore || + (isBinaryMode() && test.context.requiresDevQuarto); const userSession = !runningInCI(); const args: Deno.TestDefinition = { @@ -223,17 +227,13 @@ export function test(test: TestDescriptor) { const runTest = !test.context.prereq || await test.context.prereq(); if (runTest) { const wd = Deno.cwd(); - if (test.context?.cwd) { - Deno.chdir(test.context.cwd()); - } - if (test.context.setup) { - await test.context.setup(); - } + // The child owns log capture in binary mode. + const binMode = isBinaryMode(); let cleanedup = false; const cleanupLogOnce = async () => { - if (!cleanedup) { + if (!cleanedup && !binMode) { await cleanupLogger(); cleanedup = true; } @@ -241,8 +241,9 @@ export function test(test: TestDescriptor) { // Capture the output const log = Deno.makeTempFileSync({ suffix: ".json" }); - const handlers = await initializeLogger({ - log: test.logConfig?.log || log, + const logTarget = test.logConfig?.log || log; + const handlers = binMode ? undefined : await initializeLogger({ + log: logTarget, level: test.logConfig?.level || "INFO", format: test.logConfig?.format || "json-stream", quiet: true, @@ -258,21 +259,43 @@ export function test(test: TestDescriptor) { let lastVerify; try { + // Inside the try so a throwing setup or chdir still reaches the + // teardown and cwd restore below, instead of skipping them and + // leaking the process cwd into every later test in the file. + if (test.context?.cwd) { + Deno.chdir(test.context.cwd()); + } + + if (test.context.setup) { + await test.context.setup(); + } try { - await test.execute(); + await test.execute(logTarget); } catch (e) { - logError(e); + if (binMode) { + // Append directly because binary mode has no harness logger. + const message = e instanceof Error + ? `${e.message}\n${e.stack ?? ""}` + : String(e); + appendLogError(logTarget, message); + } else { + logError(e); + } } // Cleanup the output logging await cleanupLogOnce(); - flushLoggers(handlers); + if (handlers) { + flushLoggers(handlers); + } - // Read the output - const testOutput = logOutput(log); - if (testOutput) { + // Both logging modes write to logTarget; a missing log is a failure. + const testOutput = logOutput(logTarget); + if (testOutput === undefined) { + fail(`test log file is missing: ${logTarget}`); + } else { for (const ver of test.verify) { lastVerify = ver; if (userSession) { @@ -320,7 +343,13 @@ export function test(test: TestDescriptor) { ? colors.brightGreen(verifyFailed) : verifyFailed; - const logMessages = logOutput(log); + // Preserve the primary failure if the log is malformed. + let logMessages: ExecuteOutput[] | undefined; + try { + logMessages = logOutput(logTarget); + } catch { + logMessages = undefined; + } // Create distinctive failure marker for easy log navigation // This helps users find the failure when clicking GitHub Actions annotations @@ -358,12 +387,16 @@ export function test(test: TestDescriptor) { } finally { safeRemoveSync(log); await cleanupLogOnce(); - if (test.context.teardown) { - await test.context.teardown(); - } - - if (test.context?.cwd) { - Deno.chdir(wd); + // A throwing teardown still fails the test, but only after the cwd + // is restored - otherwise it leaks into every later test in the file. + try { + if (test.context.teardown) { + await test.context.teardown(); + } + } finally { + if (test.context?.cwd) { + Deno.chdir(wd); + } } } } else { @@ -383,6 +416,7 @@ export function test(test: TestDescriptor) { Deno.test(args); } +// Keep parsing strict; mergeChildLog() removes timeout-torn trailing records. export function readExecuteOutput(log: string) { const jsonStream = Deno.readTextFileSync(log); const lines = jsonStream.split("\n").filter((line) => !!line); diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence.test.ts diff --git a/tests/utils.ts b/tests/utils.ts index eabd8344be7..4766aa62121 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -32,6 +32,22 @@ export async function withTempDir( } } +// Runs fn with the process cwd changed to dir, restoring the original cwd +// even if fn throws. Bare Deno.chdir(dir) ... Deno.chdir(wd) leaks the cwd +// into later tests when fn rejects. +export async function withCwd( + dir: string, + fn: () => T | Promise, +): Promise { + const wd = Deno.cwd(); + Deno.chdir(dir); + try { + return await fn(); + } finally { + Deno.chdir(wd); + } +} + // Find a _quarto.yaml file in the directory hierarchy of the input file export function findProjectDir(input: string, until?: RegExp | undefined): string | undefined { let dir = dirname(input); @@ -240,8 +256,15 @@ export function fileLoader(...path: string[]) { }; } +// Resolves the quarto executable for tests that spawn a real subprocess. +// Honors QUARTO_TEST_BIN (binary mode) so these tests target the built +// quarto under test; otherwise the dev quarto from PATH. // On Windows, `quarto.cmd` needs to be explicit in `execProcess()` export function quartoDevCmd(): string { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + if (bin && bin.length > 0) { + return bin; + } return isWindows ? "quarto.cmd" : "quarto"; }