From c0e170755f03fe704041c128ee150ab019c689aa Mon Sep 17 00:00:00 2001 From: Pablo Monfort Date: Mon, 26 Oct 2026 18:41:22 -0300 Subject: [PATCH] Add Playwright text snapshot check for i18n --- .github/workflows/playwright.yml | 113 +++++ .gitignore | 6 + package-lock.json | 58 +++ package.json | 11 + playwright.config.ts | 23 + playwright/README.md | 46 ++ playwright/tests/auth.setup.ts | 14 + playwright/tests/new_work_form.spec.ts | 13 + .../new-work-form.aria.yml | 393 ++++++++++++++++++ script/playwright_prepare.rb | 11 + 10 files changed, 688 insertions(+) create mode 100644 .github/workflows/playwright.yml create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 playwright.config.ts create mode 100644 playwright/README.md create mode 100644 playwright/tests/auth.setup.ts create mode 100644 playwright/tests/new_work_form.spec.ts create mode 100644 playwright/tests/new_work_form.spec.ts-snapshots/new-work-form.aria.yml create mode 100644 script/playwright_prepare.rb diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000000..f8f8dbda993 --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,113 @@ +name: Playwright Text Snapshots +on: + pull_request: + branches: + - master + - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + playwright: + name: playwright aria snapshots + runs-on: ubuntu-latest + timeout-minutes: 20 + + env: + CI: true + RAILS_ENV: test + PLAYWRIGHT_USER: testuser + PLAYWRIGHT_PASSWORD: testuser + + services: + database: + image: mariadb:10.5.4-focal + env: + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: otwarchive_test + ports: + - 3306:3306 + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0 + ports: + - 9200:9200 + options: >- + -e="discovery.type=single-node" + -e="xpack.security.enabled=false" + + memcached: + image: memcached:1.5 + ports: + - 11211:11211 + + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Run apt-get update + run: sudo apt-get update + timeout-minutes: 2 + + - name: Install and start up redis servers + run: | + sudo apt-get install -y redis-server + ./script/gh-actions/multiple_redis.sh + timeout-minutes: 5 + + - name: Set up Ruby and run bundle install + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + + - name: Copy config files + run: | + cp config/gh-actions/database.yml config/database.yml + cp config/gh-actions/redis.yml config/redis.yml + cp config/gh-actions/local.yml config/local.yml + + - name: Set transaction isolation level + run: ./script/gh-actions/configure_database.sh + + - name: Make and seed database + run: bundle exec rake db:otwseed + + - name: Prepare elasticsearch indexes and the test user + run: bundle exec rails runner script/playwright_prepare.rb + + - name: Start the Rails server + run: | + bundle exec rails server -p 3000 & + for i in $(seq 1 60); do + curl -s -o /dev/null http://localhost:3000/ && break + sleep 2 + done + curl -s -o /dev/null -w "server responded with %{http_code}\n" http://localhost:3000/ + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Install Playwright + run: | + npm ci + npx playwright install --with-deps chromium + timeout-minutes: 10 + + - name: Run Playwright tests + run: npm run test:playwright + + - name: Upload Playwright report + if: ${{ failure() }} + uses: actions/upload-artifact@v7 + with: + name: playwright-report + path: | + playwright-report + test-results + log/test.log diff --git a/.gitignore b/.gitignore index ee17d7db4e8..47ad9447012 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,9 @@ coverage redis-cucumber-dump.rdb .rspec + +# Playwright +node_modules/ +playwright/.auth/ +playwright-report/ +test-results/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000..7a32b14442d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,58 @@ +{ + "name": "otwarchive", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "otwarchive", + "devDependencies": { + "@playwright/test": "1.64.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000000..4925966c626 --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "otwarchive", + "private": true, + "scripts": { + "test:playwright": "playwright test", + "test:playwright:update": "playwright test --update-snapshots" + }, + "devDependencies": { + "@playwright/test": "1.64.1" + } +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000000..445e2dc51d4 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,23 @@ +import { defineConfig, devices } from "@playwright/test"; + +// See playwright/README.md for how to run these tests and update baselines. +export default defineConfig({ + testDir: "./playwright/tests", + fullyParallel: false, + retries: 0, + reporter: process.env.CI ? [["list"], ["html", { open: "never" }]] : [["list"]], + use: { + baseURL: process.env.PLAYWRIGHT_BASE_URL || "http://localhost:3000", + }, + projects: [ + { + name: "setup", + testMatch: /auth\.setup\.ts/, + }, + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + dependencies: ["setup"], + }, + ], +}); diff --git a/playwright/README.md b/playwright/README.md new file mode 100644 index 00000000000..cfcb10f146e --- /dev/null +++ b/playwright/README.md @@ -0,0 +1,46 @@ +# Playwright text snapshot checks + +These tests capture [aria snapshots](https://playwright.dev/docs/aria-snapshots) +(the page's accessibility tree: roles plus visible text, stored as YAML) of +selected pages and fail with a readable text diff when the rendered text +changes. A green check means the visible text of the covered pages is +unchanged; a red one shows exactly what changed. This is useful for any +refactor that should not change what the user sees, i18n extractions being +the first use case: with the text verified, review can focus on key names +and interpolation. + +## Running locally + +The tests run against a test server seeded with `db:otwseed`, the same data +CI uses. Snapshots taken against a development database would only match +your machine. Start the server and run the tests: + +```sh +docker compose run --rm -p 3001:3000 test bash -c \ + "bundle exec rake db:otwseed && \ + bundle exec rails runner 'script/playwright_prepare.rb' && \ + bundle exec rails server -p 3000 -b 0.0.0.0" +``` + +```sh +npm install +npx playwright install chromium +PLAYWRIGHT_BASE_URL=http://localhost:3001 PLAYWRIGHT_PASSWORD=testuser \ + npm run test:playwright +``` + +## When a test fails + +- If the text change is unintentional, you found a regression. +- If it is intentional, regenerate the baselines and commit the updated + `.aria.yml` files, so the diff documents the change in the pull request. + +With the seeded test server from the section above running: + +```sh +PLAYWRIGHT_BASE_URL=http://localhost:3001 PLAYWRIGHT_PASSWORD=testuser \ + npm run test:playwright:update +``` + +Review regenerated baselines before committing: auto-generated regexes for +dynamic content (dates, counts) can be too literal and may need widening. diff --git a/playwright/tests/auth.setup.ts b/playwright/tests/auth.setup.ts new file mode 100644 index 00000000000..91cb6805a9c --- /dev/null +++ b/playwright/tests/auth.setup.ts @@ -0,0 +1,14 @@ +import { test as setup, expect } from "@playwright/test"; + +const authFile = "playwright/.auth/user.json"; +const user = process.env.PLAYWRIGHT_USER || "testuser"; +const password = process.env.PLAYWRIGHT_PASSWORD || "password"; + +setup("authenticate", async ({ page }) => { + await page.goto("/users/login"); + await page.locator("#user_login").fill(user); + await page.locator("#user_password").fill(password); + await page.locator("form#new_user").getByRole("button", { name: /log in/i }).click(); + await expect(page.locator("#main")).toContainText(/successfully logged in/i); + await page.context().storageState({ path: authFile }); +}); diff --git a/playwright/tests/new_work_form.spec.ts b/playwright/tests/new_work_form.spec.ts new file mode 100644 index 00000000000..8254c1b1d94 --- /dev/null +++ b/playwright/tests/new_work_form.spec.ts @@ -0,0 +1,13 @@ +import { test, expect } from "@playwright/test"; + +test.use({ storageState: "playwright/.auth/user.json" }); + +// The aria snapshot records the visible text of the page (roles and +// accessible names) and fails with a text diff when it changes. If the +// change is intentional, regenerate with: npm run test:playwright:update +test("new work form text is unchanged", async ({ page }) => { + await page.goto("/works/new"); + await expect(page.getByRole("main")).toMatchAriaSnapshot({ + name: "new_work_form.aria.yml", + }); +}); diff --git a/playwright/tests/new_work_form.spec.ts-snapshots/new-work-form.aria.yml b/playwright/tests/new_work_form.spec.ts-snapshots/new-work-form.aria.yml new file mode 100644 index 00000000000..c30a0074cca --- /dev/null +++ b/playwright/tests/new_work_form.spec.ts-snapshots/new-work-form.aria.yml @@ -0,0 +1,393 @@ +- main: + - heading "Post New Work" [level=2] + - list: + - listitem: + - link "Restore From Last Unposted Draft?": + - /url: /works/new?load_unposted=true + - listitem: + - link "Import From An Existing URL Instead?": + - /url: /works/new?import=true + - paragraph: "* Required information" + - group "Tags": + - text: "" + - heading "Tags" [level=3] + - paragraph: /Tags are comma separated, \d+ characters per tag\. Fandom, relationship, character, and additional tags must not add up to more than \d+\. Archive warning, category, and rating tags do not count toward this limit\./ + - term: + - text: Rating* + - link "Rating help": + - /url: /help/tags_ratings + - text: "?" + - definition: + - combobox "Rating*": + - option "Not Rated" [selected] + - option "General Audiences" + - option "Teen And Up Audiences" + - option "Mature" + - option "Explicit" + - term: + - text: Archive Warnings* + - link "Warning help": + - /url: /help/tags_warnings + - text: "?" + - definition: + - group: + - list: + - listitem: + - checkbox "Rape/Non-Con" + - text: Rape/Non-Con + - listitem: + - checkbox "Graphic Depictions Of Violence" + - text: Graphic Depictions Of Violence + - listitem: + - checkbox "Major Character Death" + - text: Major Character Death + - listitem: + - checkbox "Underage Sex" + - text: Underage Sex + - listitem: + - checkbox "Choose Not To Use Archive Warnings" + - text: Choose Not To Use Archive Warnings + - listitem: + - checkbox "No Archive Warnings Apply" + - text: No Archive Warnings Apply + - term: + - text: Fandoms* + - link "Fandom help": + - /url: /help/tags_fandoms + - text: "?" + - definition: + - list: + - listitem: + - combobox "Fandoms*" [expanded] + - paragraph: If this is the first work for a fandom, it may not show up in the fandoms page for a day or two. + - term: + - text: Categories + - link "Categories help": + - /url: /help/tags_categories + - text: "?" + - definition: + - group: + - list: + - listitem: + - checkbox "F/F" + - text: F/F + - listitem: + - checkbox "F/M" + - text: F/M + - listitem: + - checkbox "Gen" + - text: Gen + - listitem: + - checkbox "M/M" + - text: M/M + - listitem: + - checkbox "Multi" + - text: Multi + - listitem: + - checkbox "Other" + - text: Other + - term: + - text: Relationships + - link "Relationships help": + - /url: /help/tags_relationships + - text: "?" + - definition: + - list: + - listitem: + - combobox "Relationships" [expanded] + - term: + - text: Characters + - link "Characters help": + - /url: /help/tags_characters + - text: "?" + - definition: + - list: + - listitem: + - combobox "Characters" [expanded] + - term: + - text: Additional Tags + - link "Additional tags help": + - /url: /help/tags_additional + - text: "?" + - definition: + - list: + - listitem: + - combobox "Additional Tags" [expanded] + - article "Preface": + - text: "" + - heading "Preface" [level=3] + - term: Work Title* + - definition: + - textbox "Work Title*" + - alert + - paragraph: /\d+ characters left/ + - term: Creator/Pseud(s) + - definition: + - listbox "Creator/Pseud(s)": + - option "Cal" [selected] + - option "Non-Default Pseud" + - option "Some other fucking pseud" + - option "testuser" + - option "thelongestpseudinthegoshdarnworldomgwtfo" + - term: Add co-creators? + - definition: + - checkbox "Add co-creators?" + - group "Add co-creators": + - list: + - listitem: + - combobox [expanded] + - term: Summary + - definition: + - textbox "Summary" + - alert + - paragraph: /\d+ characters left/ + - term: Notes + - definition: + - list: + - listitem: + - checkbox "at the beginning" + - text: "at the beginning Warning: Unchecking this box will delete the existing beginning note." + - group "Notes": + - text: "" + - textbox "Notes" + - paragraph: /\d+ characters left/ + - listitem: + - checkbox "at the end" + - text: "at the end Warning: Unchecking this box will delete the existing end note." + - group "End Notes": + - text: "" + - textbox "End Notes" + - paragraph: /\d+ characters left/ + - article "Associations": + - text: "" + - heading "Associations" [level=3] + - term: + - text: Post to Collections / Challenges + - link "Adding To Collections": + - /url: /help/collectibles_add_to_collection + - text: "?" + - definition: + - list: + - listitem: + - combobox "Post to Collections / Challenges" [expanded] + - term: + - text: Gift this work to + - link "Recipients": + - /url: /help/works_recipients + - text: "?" + - definition: + - list: + - listitem: + - combobox "Gift this work to" [expanded] + - term: + - checkbox "This work is a remix, a translation, a podfic, or was inspired by another work" + - definition: + - text: This work is a remix, a translation, a podfic, or was inspired by another work + - link "Parent works help": + - /url: /help/works_parents + - text: "?" + - group: + - term: URL + - definition: + - textbox "URL" + - paragraph: For a work in the Archive, only the URL is required. + - term: Title + - definition: + - textbox "Title" + - term: Author + - definition: + - textbox "Author" + - term: + - text: Language + - link "Languages help": + - /url: /help/works_languages + - text: "?" + - definition: + - combobox "Language": + - option [selected] + - option ">中文" + - option "Bahasa Indonesia" + - option "Čeština" + - option "Deutsch" + - option "English" + - option "Español" + - option "Français" + - option "Italiano" + - option "Nederlands" + - option "日本語" + - option "Português" + - option "Русский" + - option "Suomi" + - term: + - checkbox "This is a translation" + - definition: + - text: This is a translation + - link "Translation link": + - /url: /help/works_translation_link + - text: "?" + - term: + - checkbox "This work is part of a series" + - definition: + - text: This work is part of a series + - link "Choosing series": + - /url: /help/works_series + - text: "?" + - group: + - term: "Choose one of your existing series:" + - definition: + - combobox "Choose one of your existing series:": + - option "Please select" [selected] + - option "Test Series" + - term: "Or create and use a new one:" + - definition: + - textbox "Or create and use a new one:" + - term: + - checkbox "This work has multiple chapters" + - definition: + - text: This work has multiple chapters + - group: + - term: Chapter 1 of + - definition: + - textbox "Chapter 1 of" + - term: "Chapter Title:" + - definition: + - textbox "Chapter Title:" + - term: + - checkbox "Set a different publication date" + - definition: + - text: Set a different publication date + - link "Publication Date Options": + - /url: /help/works_backdating + - text: "?" + - group: + - term: Set publication date + - definition: + - combobox + - combobox: + - option "January" + - option "February" + - option "March" + - option "April" + - option "May" + - option "June" + - option "July" + - option "August" + - option "September" + - option "October" + - option "November" + - option "December" + - combobox + - term: + - text: Choose a language * + - link "Languages help": + - /url: /help/works_languages + - text: "?" + - definition: + - combobox "Choose a language *": + - option "Please select a language" [selected] + - option ">中文" + - option "Bahasa Indonesia" + - option "Čeština" + - option "Deutsch" + - option "English" + - option "Español" + - option "Français" + - option "Italiano" + - option "Nederlands" + - option "日本語" + - option "Português" + - option "Русский" + - option "Suomi" + - term: + - text: Select work skin + - link "Work skins": + - /url: /help/works_skins + - text: "?" + - definition: + - combobox "Select work skin": + - option [selected] + - navigation: + - link "Public Work Skins": + - /url: /skins?skin_type=WorkSkin + - article "Privacy": + - text: "" + - heading "Privacy" [level=3] + - term: + - checkbox "Only show your work to registered users" + - definition: + - text: Only show your work to registered users + - link "Registered users": + - /url: /help/privacy_restricted_work + - text: "?" + - term: + - checkbox "Enable comment moderation" + - definition: + - text: Enable comment moderation + - link "Comments moderated": + - /url: /help/privacy_moderated_commenting + - text: "?" + - term: + - text: Who can comment on this work + - link "Who can comment on this work": + - /url: /help/privacy_restricted_commenting + - text: "?" + - definition: + - group: + - list: + - listitem: + - radio "Registered users and guests can comment" + - text: Registered users and guests can comment + - listitem: + - radio "Only registered users can comment" [checked] + - text: Only registered users can comment + - listitem: + - radio "No one can comment" + - text: No one can comment + - article "Work Text*": + - text: "" + - menu: + - listitem: + - link "Rich Text": + - /url: "#" + - listitem: + - link "HTML": + - /url: "#" + - paragraph: + - text: Plain text with limited HTML + - link "Html help": + - /url: /help/html + - text: "?" + - text: Type or paste formatted text. + - link "Rte help": + - /url: /help/rte + - text: "?" + - paragraph: + - strong: "Note:" + - text: Text entered in the posting form is + - strong: not + - text: automatically saved. Always keep a backup copy of your work. + - heading "Work Text*" [level=4] + - textbox + - alert + - paragraph: /\d+ characters left/ + - group "Post Work": + - text: "" + - paragraph: + - text: All works on AO3 must comply with our + - link "Content Policy": + - /url: /content + - text: . For more information, please refer to our + - link "Terms of Service FAQ": + - /url: /tos_faq#content_faq + - text: . + - list: + - listitem: + - button "Save Draft" + - listitem: + - button "Preview" + - listitem: + - button "Post" + - listitem: + - link "Cancel": + - /url: /users/testuser/works + diff --git a/script/playwright_prepare.rb b/script/playwright_prepare.rb new file mode 100644 index 00000000000..1200544db77 --- /dev/null +++ b/script/playwright_prepare.rb @@ -0,0 +1,11 @@ +# Prepares a freshly seeded database for the Playwright tests: creates the +# elasticsearch indexes and accepts the current TOS for the login user, so +# the TOS prompt doesn't cover the pages under test. +Indexer.all.map(&:prepare_for_testing) + +controller = ApplicationController.new +controller.load_tos_popup +tos_version = controller.instance_variable_get(:@current_tos_version) + +login = ENV.fetch("PLAYWRIGHT_USER", "testuser") +User.find_by!(login: login).update_column(:accepted_tos_version, tos_version)