From 87198d59b79af3464e088318d792e344e77fc0ad Mon Sep 17 00:00:00 2001 From: Chris Kenst Date: Fri, 28 Aug 2026 11:35:53 -0700 Subject: [PATCH] Add regression tests for conference data validator Add a Minitest suite (test/validate_data_test.rb) with fixture data directories covering valid data, representative errors, warnings, and date-parsing edge cases for tools/validate_data.rb. The validator now honors a VALIDATE_DATA_ROOT env var so tests can point at fixtures without touching the real _data/ files. Wire the suite into CI so it runs alongside the existing validator invocation. Closes the Validator Regression Tests item in ROADMAP.md Phase 1.5. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 3 + AGENTS.md | 2 + ROADMAP.md | 2 +- test/fixtures/dates/_data/closed.yml | 5 + test/fixtures/dates/_data/current.yml | 24 +++++ test/fixtures/dates/_data/past.yml | 4 + test/fixtures/errors/_data/closed.yml | 5 + test/fixtures/errors/_data/current.yml | 20 ++++ test/fixtures/errors/_data/past.yml | 4 + test/fixtures/valid/_data/closed.yml | 5 + test/fixtures/valid/_data/current.yml | 11 ++ test/fixtures/valid/_data/past.yml | 5 + test/fixtures/warnings/_data/closed.yml | 5 + test/fixtures/warnings/_data/current.yml | 29 +++++ test/fixtures/warnings/_data/past.yml | 4 + test/validate_data_test.rb | 132 +++++++++++++++++++++++ tools/validate_data.rb | 2 +- 17 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/dates/_data/closed.yml create mode 100644 test/fixtures/dates/_data/current.yml create mode 100644 test/fixtures/dates/_data/past.yml create mode 100644 test/fixtures/errors/_data/closed.yml create mode 100644 test/fixtures/errors/_data/current.yml create mode 100644 test/fixtures/errors/_data/past.yml create mode 100644 test/fixtures/valid/_data/closed.yml create mode 100644 test/fixtures/valid/_data/current.yml create mode 100644 test/fixtures/valid/_data/past.yml create mode 100644 test/fixtures/warnings/_data/closed.yml create mode 100644 test/fixtures/warnings/_data/current.yml create mode 100644 test/fixtures/warnings/_data/past.yml create mode 100644 test/validate_data_test.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcfa84f9..c6ba59bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,9 @@ jobs: - name: Validate conference data run: ruby tools/validate_data.rb + - name: Run validator regression tests + run: ruby test/validate_data_test.rb + - name: Build with Jekyll run: bundle exec jekyll build --verbose diff --git a/AGENTS.md b/AGENTS.md index 865153e2..abdde401 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,6 +97,7 @@ Non-Docker build commands: ```bash bundle install ruby tools/validate_data.rb +ruby test/validate_data_test.rb bundle exec jekyll build --verbose bundle exec htmlproofer ./_site --disable-external --no-enforce-https --allow-missing-href --ignore-urls '/^\\/\\//' ``` @@ -109,6 +110,7 @@ Before finishing a change, check: - YAML parses correctly. - `ruby tools/validate_data.rb` passes. +- `ruby test/validate_data_test.rb` passes when the validator itself is changed. - Jekyll builds successfully when the environment supports it. - Conference URLs include `utm_source=testingconferences` where appropriate. - `twitter` values do not include `@`. diff --git a/ROADMAP.md b/ROADMAP.md index 7cfa604e..c273b2af 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -15,7 +15,7 @@ This is the single source of truth for planned project work. Keep goals, status, The July 2026 readiness assessment found the repository ready for supervised conference-data changes. The remaining work is primarily deployment clarity, maintenance-tool documentation, validator regression coverage, formatting expectations, and reliable local verification. - [x] **Workflow Simplification**: Simplify or rename `.github/workflows/deploy.yml` so it matches the confirmed model where GitHub Pages deploys from `main`. -- [ ] **Validator Regression Tests**: Add focused fixtures and automated tests for valid data, representative failures, warnings, date parsing, and schema changes. +- [x] **Validator Regression Tests**: Add focused fixtures and automated tests for valid data, representative failures, warnings, date parsing, and schema changes. - [ ] **Branch Protection**: Require the consolidated validation check for `main` and document the expectation. - [ ] **Contributor Support Cleanup**: Replace the Code of Conduct placeholder and align contributor support text with the issue forms that actually exist. diff --git a/test/fixtures/dates/_data/closed.yml b/test/fixtures/dates/_data/closed.yml new file mode 100644 index 00000000..fceddbda --- /dev/null +++ b/test/fixtures/dates/_data/closed.yml @@ -0,0 +1,5 @@ +- name: Closed Date Range Conference + location: Nowhere, USA + first_date: "January 1, 2015" + last_date: "January 3, 2018" + url: https://example.com/closed-dates?utm_source=testingconferences diff --git a/test/fixtures/dates/_data/current.yml b/test/fixtures/dates/_data/current.yml new file mode 100644 index 00000000..a8e41383 --- /dev/null +++ b/test/fixtures/dates/_data/current.yml @@ -0,0 +1,24 @@ +- name: Single Day Conference + location: Nowhere, USA + dates: "March 5, 2026" + url: https://example.com/single-day?utm_source=testingconferences + +- name: Same Month Range Conference + location: Nowhere, USA + dates: "April 6-8, 2026" + url: https://example.com/same-month?utm_source=testingconferences + +- name: Cross Month Range Conference + location: Nowhere, USA + dates: "April 30 - May 2, 2026" + url: https://example.com/cross-month?utm_source=testingconferences + +- name: Reversed Day Month Range Conference + location: Nowhere, USA + dates: "April 29-1 May, 2026" + url: https://example.com/reversed-day-month?utm_source=testingconferences + +- name: Abbreviated Month Conference + location: Nowhere, USA + dates: "Sept 10, 2026" + url: https://example.com/abbreviated-month?utm_source=testingconferences diff --git a/test/fixtures/dates/_data/past.yml b/test/fixtures/dates/_data/past.yml new file mode 100644 index 00000000..ded54405 --- /dev/null +++ b/test/fixtures/dates/_data/past.yml @@ -0,0 +1,4 @@ +- name: Dates Past Placeholder + location: Nowhere, USA + dates: "January 1, 2020" + url: https://example.com/dates-past?utm_source=testingconferences diff --git a/test/fixtures/errors/_data/closed.yml b/test/fixtures/errors/_data/closed.yml new file mode 100644 index 00000000..03dc496f --- /dev/null +++ b/test/fixtures/errors/_data/closed.yml @@ -0,0 +1,5 @@ +- name: Reversed Dates Workshop + location: Nowhere, USA + first_date: "January 1, 2020" + last_date: "January 1, 2015" + url: https://example.com/reversed?utm_source=testingconferences diff --git a/test/fixtures/errors/_data/current.yml b/test/fixtures/errors/_data/current.yml new file mode 100644 index 00000000..f4aa21a7 --- /dev/null +++ b/test/fixtures/errors/_data/current.yml @@ -0,0 +1,20 @@ +- location: Nowhere, USA + dates: "May 1, 2026" + url: https://example.com/missing-name?utm_source=testingconferences + +- name: Bad URL Conference + location: Nowhere, USA + dates: "May 2, 2026" + url: not-a-valid-url + +- name: At Symbol Conference + location: Nowhere, USA + dates: "May 3, 2026" + url: https://example.com/at-symbol?utm_source=testingconferences + twitter: "@atsymbol" + +- name: Unknown Field Conference + location: Nowhere, USA + dates: "May 4, 2026" + url: https://example.com/unknown-field?utm_source=testingconferences + organizer: Someone diff --git a/test/fixtures/errors/_data/past.yml b/test/fixtures/errors/_data/past.yml new file mode 100644 index 00000000..31de4a45 --- /dev/null +++ b/test/fixtures/errors/_data/past.yml @@ -0,0 +1,4 @@ +- name: Placeholder Past Conference + location: Nowhere, USA + dates: "May 1, 2020" + url: https://example.com/past-placeholder diff --git a/test/fixtures/valid/_data/closed.yml b/test/fixtures/valid/_data/closed.yml new file mode 100644 index 00000000..42127e24 --- /dev/null +++ b/test/fixtures/valid/_data/closed.yml @@ -0,0 +1,5 @@ +- name: Delta Legacy Workshop + location: Chicago, IL, USA + first_date: "January 1, 2015" + last_date: "January 3, 2018" + url: https://example.com/delta?utm_source=testingconferences diff --git a/test/fixtures/valid/_data/current.yml b/test/fixtures/valid/_data/current.yml new file mode 100644 index 00000000..48f8767e --- /dev/null +++ b/test/fixtures/valid/_data/current.yml @@ -0,0 +1,11 @@ +- name: Alpha Testing Conference 2026 + location: Austin, TX, USA + dates: "March 1-3, 2026" + url: https://example.com/alpha-2026?utm_source=testingconferences + twitter: alphatest + status: Registration is Open + +- name: Beta Quality Summit 2026 + location: Online + dates: "June 10-11, 2026" + url: https://example.com/beta-2026?utm_source=testingconferences diff --git a/test/fixtures/valid/_data/past.yml b/test/fixtures/valid/_data/past.yml new file mode 100644 index 00000000..e5cb32a6 --- /dev/null +++ b/test/fixtures/valid/_data/past.yml @@ -0,0 +1,5 @@ +- name: Gamma Test Days 2024 + location: Berlin, Germany + dates: "April 5-6, 2024" + url: https://example.com/gamma-2024 + video_playlist: https://example.com/gamma-2024/playlist diff --git a/test/fixtures/warnings/_data/closed.yml b/test/fixtures/warnings/_data/closed.yml new file mode 100644 index 00000000..2469d3d4 --- /dev/null +++ b/test/fixtures/warnings/_data/closed.yml @@ -0,0 +1,5 @@ +- name: Closed Placeholder For Warnings + location: Nowhere, USA + first_date: "January 1, 2015" + last_date: "January 1, 2016" + url: https://example.com/closed-warnings?utm_source=testingconferences diff --git a/test/fixtures/warnings/_data/current.yml b/test/fixtures/warnings/_data/current.yml new file mode 100644 index 00000000..3ce4f4a5 --- /dev/null +++ b/test/fixtures/warnings/_data/current.yml @@ -0,0 +1,29 @@ +- name: No Tracking Conference + location: Nowhere, USA + dates: "January 1, 2026" + url: https://example.com/no-tracking + +- name: Out Of Order Conference + location: Nowhere, USA + dates: "January 1, 2026" + url: https://example.com/first?utm_source=testingconferences + +- name: Earlier Conference + location: Nowhere, USA + dates: "January 1, 2025" + url: https://example.com/second?utm_source=testingconferences + +- name: Duplicate Name Conference + location: Nowhere, USA + dates: "February 1, 2026" + url: https://example.com/dup-1?utm_source=testingconferences + +- name: Duplicate Name Conference + location: Nowhere, USA + dates: "February 2, 2026" + url: https://example.com/dup-2?utm_source=testingconferences + +- name: Unparsable Date Conference + location: Nowhere, USA + dates: "Sometime next spring" + url: https://example.com/unparsable?utm_source=testingconferences diff --git a/test/fixtures/warnings/_data/past.yml b/test/fixtures/warnings/_data/past.yml new file mode 100644 index 00000000..fd70bd11 --- /dev/null +++ b/test/fixtures/warnings/_data/past.yml @@ -0,0 +1,4 @@ +- name: Past Placeholder For Warnings + location: Nowhere, USA + dates: "January 1, 2020" + url: https://example.com/past-warnings?utm_source=testingconferences diff --git a/test/validate_data_test.rb b/test/validate_data_test.rb new file mode 100644 index 00000000..1a85f2e9 --- /dev/null +++ b/test/validate_data_test.rb @@ -0,0 +1,132 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true +# +# Regression tests for tools/validate_data.rb. +# +# These tests run the validator as a subprocess against fixture data +# directories under test/fixtures/*/_data, so they exercise the exact same +# code path as CI and contributors (`ruby tools/validate_data.rb`) without +# ever touching the real _data/ files. +# +# Run with: ruby test/validate_data_test.rb + +require 'minitest/autorun' +require 'open3' + +ROOT = File.expand_path('..', __dir__) +VALIDATOR = File.join(ROOT, 'tools', 'validate_data.rb') + +def run_validator(fixture_name) + fixture_root = File.join(ROOT, 'test', 'fixtures', fixture_name) + env = { 'VALIDATE_DATA_ROOT' => fixture_root } + stdout, stderr, status = Open3.capture3(env, 'ruby', VALIDATOR) + [stdout, stderr, status] +end + +class ValidateDataTest < Minitest::Test + def test_valid_fixtures_pass_with_no_warnings_or_errors + stdout, stderr, status = run_validator('valid') + + assert status.success?, "expected success, got stderr:\n#{stderr}" + assert_includes stdout, 'passed with 0 warning(s)' + assert_empty stderr + end + + def test_missing_required_field_is_an_error + _stdout, stderr, status = run_validator('errors') + + refute status.success? + assert_match(/missing required field `name`/, stderr) + end + + def test_invalid_url_is_an_error + _stdout, stderr, status = run_validator('errors') + + assert_match(/url is not a valid HTTP\(S\) URL/, stderr) + end + + def test_twitter_handle_with_at_symbol_is_an_error + _stdout, stderr, status = run_validator('errors') + + assert_match(/twitter value should not include @/, stderr) + end + + def test_unknown_field_in_strict_file_is_an_error + _stdout, stderr, status = run_validator('errors') + + assert_match(/unknown fields: organizer/, stderr) + end + + def test_first_date_after_last_date_is_an_error + _stdout, stderr, status = run_validator('errors') + + assert_match(/first_date is after last_date/, stderr) + end + + def test_missing_tracking_source_is_a_warning_not_an_error + _stdout, stderr, status = run_validator('warnings') + + assert_match(/WARNING:.*missing utm_source=testingconferences/, stderr) + refute_match(/ERROR:.*missing utm_source=testingconferences/, stderr) + end + + def test_out_of_order_dates_is_a_warning + _stdout, stderr, status = run_validator('warnings') + + assert_match(/WARNING:.*appears out of chronological order/, stderr) + end + + def test_duplicate_name_in_same_file_is_a_warning + _stdout, stderr, status = run_validator('warnings') + + assert_match(/WARNING:.*duplicate name in _data\/current\.yml/, stderr) + end + + def test_unparsable_date_is_a_warning + _stdout, stderr, status = run_validator('warnings') + + assert_match(/WARNING:.*could not parse dates for ordering/, stderr) + end + + def test_warnings_fixture_still_exits_success + _stdout, _stderr, status = run_validator('warnings') + + assert status.success?, 'warnings alone should not fail validation' + end + + def test_single_day_date_parses_without_warning + _stdout, stderr, _status = run_validator('dates') + + refute_match(/Single Day Conference.*could not parse/, stderr) + end + + def test_same_month_date_range_parses_without_warning + _stdout, stderr, _status = run_validator('dates') + + refute_match(/Same Month Range Conference.*could not parse/, stderr) + end + + def test_cross_month_date_range_parses_without_warning + _stdout, stderr, _status = run_validator('dates') + + refute_match(/Cross Month Range Conference.*could not parse/, stderr) + end + + def test_reversed_day_month_range_parses_without_warning + _stdout, stderr, _status = run_validator('dates') + + refute_match(/Reversed Day Month Range Conference.*could not parse/, stderr) + end + + def test_abbreviated_month_parses_without_warning + _stdout, stderr, _status = run_validator('dates') + + refute_match(/Abbreviated Month Conference.*could not parse/, stderr) + end + + def test_closed_yml_date_range_parses + _stdout, stderr, _status = run_validator('dates') + + refute_match(/Closed Date Range Conference.*could not parse/, stderr) + end +end diff --git a/tools/validate_data.rb b/tools/validate_data.rb index 4108b4ae..df2bcd08 100644 --- a/tools/validate_data.rb +++ b/tools/validate_data.rb @@ -5,7 +5,7 @@ require 'uri' require 'yaml' -ROOT = File.expand_path('..', __dir__) +ROOT = ENV['VALIDATE_DATA_ROOT'] || File.expand_path('..', __dir__) DATA_FILES = { '_data/current.yml' => {