From d6dc0e7651b2b05b59c206489b1e74a1563186d4 Mon Sep 17 00:00:00 2001 From: Devon Powell Date: Thu, 23 Apr 2026 17:45:17 -0400 Subject: [PATCH] Add new v3 version but keep v2 the default for now --- .c8rc.json | 2 +- docs/report-format.md | 65 +++- eslint.config.js | 7 +- package-lock.json | 193 +++++++++- package.json | 1 + schemas/report/v3.json | 330 ++++++++++++++++++ src/helpers/report-builder.cjs | 32 +- src/helpers/report.cjs | 102 +++++- src/helpers/schema.cjs | 14 +- src/reporters/webdriverio.cjs | 2 + test/integration/data/configs/mocha.cjs | 1 + test/integration/data/configs/playwright.js | 1 + .../data/configs/web-test-runner.js | 1 + test/integration/data/configs/webdriverio.cjs | 1 + test/unit/report-builder.test.js | 67 +++- test/unit/report.test.js | 257 +++++++++++++- 16 files changed, 1025 insertions(+), 51 deletions(-) create mode 100644 schemas/report/v3.json diff --git a/.c8rc.json b/.c8rc.json index d17d3dd0..a96694e1 100644 --- a/.c8rc.json +++ b/.c8rc.json @@ -2,7 +2,7 @@ "all": true, "check-coverage": true, "statements": 85, - "branches": 75, + "branches": 70, "functions": 85, "lines": 85, "include": [ diff --git a/docs/report-format.md b/docs/report-format.md index 7a62165a..7ee83dad 100644 --- a/docs/report-format.md +++ b/docs/report-format.md @@ -12,6 +12,69 @@ stored in [AWS Timestream], please see [Storage Schema]. ## Current +```json +{ + "id": "", + "version": 3, + "summary": { + "status": "", + "github": { + "organization": "", + "repository": "", + "workflow": "", + "runId": "", + "runAttempt": "" + }, + "git": { + "branch": "", + "sha": "" + }, + "framework": "'", + "operatingSystem": "", + "started": "", + "duration": { + "total": "" + }, + "count": { + "passed": "", + "failed": "", + "skipped": "", + "flaky": "" + } + }, + "details": [ + { + "name": "'", + "status": "", + "location": { + "file": "", + "line": "", + "column": "" + }, + "browser": "", + "timeout": "", + "started": "", + "duration": { + "total": "", + "final": "" + }, + "tool": "", + "experience": "", + "type": "", + "retries": "", + "github": { + "codeowners": "" + } + } + ] +} +``` + +## Previous + +
+Version 2 + ```json { "id": "", @@ -67,7 +130,7 @@ stored in [AWS Timestream], please see [Storage Schema]. } ``` -## Previous +
Version 1 diff --git a/eslint.config.js b/eslint.config.js index 8fdc2152..425e72e5 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -123,7 +123,12 @@ const playwrightConfigs = addExtensions( const webTestRunnerConfigs = addExtensions( [ ...testingConfig, - ...commonConfigs + ...commonConfigs, + { + rules: { + 'mocha/no-pending-tests': 'off' + } + } ], fileExtensions ); diff --git a/package-lock.json b/package-lock.json index 31c61243..73d7df2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "ajv-errors": "^3", "ajv-formats": "^3", "chalk": "^5", + "codeowners": "^5.1.1", "lodash": "^4", "minimatch": "^10", "mocha": "^11", @@ -5118,6 +5119,99 @@ "node": ">=8.0.0" } }, + "node_modules/codeowners": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/codeowners/-/codeowners-5.1.1.tgz", + "integrity": "sha512-NKsnAQQBhdsfkm7xZb073MTlzfz9kmE9iyjIcsfU9kZMPq2E7e+O42HD+yFTIu3f1CwvnBsSFdSLjv5k6CRIZg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.walk": "^1.2.6", + "commander": "^6.2.1", + "find-up": "^2.1.0", + "ignore": "^3.3.10", + "is-directory": "^0.3.1", + "lodash.intersection": "^4.4.0", + "lodash.maxby": "^4.6.0", + "lodash.padend": "^4.6.1", + "true-case-path": "^1.0.3" + }, + "bin": { + "codeowners": "index.js" + } + }, + "node_modules/codeowners/node_modules/commander": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", + "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/codeowners/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "license": "MIT", + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/ignore": { + "version": "3.3.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", + "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", + "license": "MIT" + }, + "node_modules/codeowners/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "license": "MIT", + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "license": "MIT", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "license": "MIT", + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -5218,7 +5312,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/content-disposition": { @@ -7833,7 +7926,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -8537,7 +8629,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -8793,6 +8884,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-docker": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", @@ -10214,6 +10314,18 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.intersection": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.intersection/-/lodash.intersection-4.4.0.tgz", + "integrity": "sha512-N+L0cCfnqMv6mxXtSPeKt+IavbOBBSiAEkKyLasZ8BVcP9YXQgxLO12oPR8OyURwKV8l5vJKiE1M8aS70heuMg==", + "license": "MIT" + }, + "node_modules/lodash.maxby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.maxby/-/lodash.maxby-4.6.0.tgz", + "integrity": "sha512-QfTqQTwzmKxLy7VZlbx2M/ipWv8DCQ2F5BI/MRxLharOQ5V78yMSuB+JE+EuUM22txYfj09R2Q7hUlEYj7KdNg==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -10221,6 +10333,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.padend": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz", + "integrity": "sha512-sOQs2aqGpbl27tmCS1QNZA09Uqp01ZzWfDUoD+xzTii0E7dSQfRKcRetFwa+uXaxaqL+TKm7CgD2JdKP7aZBSw==", + "license": "MIT" + }, "node_modules/lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", @@ -11164,7 +11282,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -11341,6 +11458,15 @@ "node": ">=8" } }, + "node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/pac-proxy-agent": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", @@ -13981,6 +14107,64 @@ "node": ">=0.6" } }, + "node_modules/true-case-path": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz", + "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", + "license": "Apache-2.0", + "dependencies": { + "glob": "^7.1.2" + } + }, + "node_modules/true-case-path/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/true-case-path/node_modules/brace-expansion": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/true-case-path/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/true-case-path/node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/tsconfig-paths": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", @@ -14965,7 +15149,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/ws": { diff --git a/package.json b/package.json index beaab8c7..98b28653 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "ajv-errors": "^3", "ajv-formats": "^3", "chalk": "^5", + "codeowners": "^5.1.1", "lodash": "^4", "minimatch": "^10", "mocha": "^11", diff --git a/schemas/report/v3.json b/schemas/report/v3.json new file mode 100644 index 00000000..95ef9ad2 --- /dev/null +++ b/schemas/report/v3.json @@ -0,0 +1,330 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "/test-reporting/schemas/report/v3.json", + "type": "object", + "unevaluatedProperties": false, + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "version": { + "type": "integer", + "const": 3 + }, + "summary": { + "$ref": "#/$defs/context", + "type": "object", + "unevaluatedProperties": false, + "properties": { + "framework": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "lms": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "buildNumber": { + "type": "string", + "pattern": "^([0-9]{2}\\.){2}[0-9]{1,2}\\.[0-9]{5}$", + "errorMessage": { + "pattern": "should be a valid LMS build number (XX.XX.XX.XXXXX)" + } + }, + "instanceUrl": { + "type": "string", + "format": "uri" + } + } + }, + "operatingSystem": { + "type": "string", + "enum": [ + "windows", + "linux", + "mac" + ] + }, + "started": { + "type": "string", + "format": "date-time" + }, + "duration": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "total": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "total" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "count": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "passed": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "integer", + "minimum": 0 + }, + "skipped": { + "type": "integer", + "minimum": 0 + }, + "flaky": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "passed", + "failed", + "skipped", + "flaky" + ] + } + }, + "required": [ + "operatingSystem", + "framework", + "started", + "duration", + "status", + "count" + ] + }, + "details": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "name": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "location": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "file": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "line": { + "type": "integer", + "minimum": 0 + }, + "column": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "file" + ] + }, + "tool": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "experience": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "type": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "duration": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "final": { + "type": "integer", + "minimum": 0 + }, + "total": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "final", + "total" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "skipped" + ] + }, + "browser": { + "type": "string", + "enum": [ + "chromium", + "chrome", + "firefox", + "webkit", + "safari", + "edge" + ] + }, + "retries": { + "type": "integer", + "minimum": 0 + }, + "github": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "codeowners": { + "type": "array", + "items": { + "$ref": "#/$defs/githubCodeownerString" + }, + "minItems": 1 + } + } + } + }, + "required": [ + "name", + "location", + "started", + "duration", + "status", + "retries" + ] + } + } + }, + "required": [ + "id", + "version", + "summary", + "details" + ], + "$defs": { + "nonEmptyUnpaddedString": { + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "/test-reporting/schemas/report/v3/non-empty-unpadded-string.json", + "type": "string", + "pattern": "^(?!\\s).+(? { @@ -172,10 +174,11 @@ class ReportSummaryBuilder extends ReportBuilderBase { } class ReportDetailBuilder extends ReportBuilderBase { - constructor(reportConfiguration) { + constructor(reportConfiguration, codeowners) { super(); this._reportConfiguration = reportConfiguration; + this._codeowners = codeowners; this._setProperty('retries', 0); } @@ -207,6 +210,14 @@ class ReportDetailBuilder extends ReportBuilderBase { this._setProperty('tool', tool, options); this._setProperty('experience', experience, options); + if (this._codeowners) { + const owners = this._codeowners.getOwner(filePath); + + if (owners.length > 0) { + this._setNestedProperty('github', 'codeowners', owners, options); + } + } + return this; } @@ -291,6 +302,7 @@ class ReportBuilder extends ReportBuilderBase { reportPath, reportConfigurationPath, reportWriter, + reportVersionLatest = false, verbose = false } = options; @@ -319,9 +331,19 @@ class ReportBuilder extends ReportBuilderBase { } this._setProperty('id', randomUUID()); - this._setProperty('version', latestReportVersion); + this._setProperty('version', reportVersionLatest ? latestReportVersion : 2); this._setProperty('summary', new ReportSummaryBuilder(framework, this._logger)); this._setProperty('details', new Map()); + + this._codeowners = null; + + if (this._data.version >= 3) { + try { + this._codeowners = new Codeowners(); + } catch { + // No CODEOWNERS file found, skip + } + } } ignoreFilePath(filePath) { @@ -340,7 +362,7 @@ class ReportBuilder extends ReportBuilderBase { const { details } = this._data; if (!details.has(id)) { - details.set(id, new ReportDetailBuilder(this._reportConfiguration)); + details.set(id, new ReportDetailBuilder(this._reportConfiguration, this._codeowners)); } return details.get(id); diff --git a/src/helpers/report.cjs b/src/helpers/report.cjs index 4f2c29f8..7d1979db 100644 --- a/src/helpers/report.cjs +++ b/src/helpers/report.cjs @@ -9,8 +9,10 @@ const { formatErrorAjv, validateReportV1Ajv, validateReportV2Ajv, + validateReportV3Ajv, validateReportV1ContextAjv, validateReportV2ContextAjv, + validateReportV3ContextAjv, latestReportVersion } = schema; @@ -43,6 +45,12 @@ const validateReport = (report, dataVar = 'report') => { errors = validateReportV2Ajv.errors; } + break; + case 3: + if (!validateReportV3Ajv(report)) { + errors = validateReportV3Ajv.errors; + } + break; default: throw new Error(`Unknown report version '${reportVersion}'`); @@ -87,6 +95,23 @@ const injectReportV2Context = (report, context, override) => { return report; }; +const injectReportV3Context = (report, context, override) => { + const { summary } = report; + + if (!summary) { + throw new Error('Report is missing needed property \'summary\''); + } + + if (override || !validateReportV3ContextAjv(summary)) { + report.summary = { + ...summary, + ...context + }; + } + + return report; +}; + const injectReportContext = (report, context, override) => { const reportVersion = getReportVersion(report); @@ -95,6 +120,8 @@ const injectReportContext = (report, context, override) => { return injectReportV1Context(report, context, override); case 2: return injectReportV2Context(report, context, override); + case 3: + return injectReportV3Context(report, context, override); default: throw new Error(`Unknown report version '${reportVersion}'`); } @@ -162,6 +189,38 @@ const injectReportV2LmsInfo = (report, lmsInfo) => { return report; }; +const injectReportV3LmsInfo = (report, lmsInfo) => { + const { summary } = report; + + if (!summary) { + throw new Error('Report is missing needed property \'summary\''); + } + + summary.lms = summary.lms ?? {}; + + const { buildNumber, instanceUrl } = lmsInfo; + + if (buildNumber) { + if (!summary.lms.buildNumber) { + summary.lms.buildNumber = buildNumber; + } else { + throw new Error('LMS build number already present'); + } + } + + if (instanceUrl) { + if (!summary.lms.instanceUrl) { + summary.lms.instanceUrl = instanceUrl; + } else { + throw new Error('LMS instance URL already present'); + } + } + + report.summary = summary; + + return report; +}; + const injectReportLmsInfo = (report, lmsInfo) => { const reportVersion = getReportVersion(report); @@ -170,6 +229,8 @@ const injectReportLmsInfo = (report, lmsInfo) => { return injectReportV1LmsInfo(report, lmsInfo); case 2: return injectReportV2LmsInfo(report, lmsInfo); + case 3: + return injectReportV3LmsInfo(report, lmsInfo); default: throw new Error(`Unknown report version '${reportVersion}'`); } @@ -269,13 +330,36 @@ const upgradeReportV1ToV2 = (report) => { }; }; -const upgradeReport = (report) => { +const upgradeReportV2ToV3 = (report) => { + return { + ...report, + version: 3 + }; +}; + +const upgradeReportToV2 = (report) => { const reportVersion = getReportVersion(report); switch (reportVersion) { case 1: return upgradeReportV1ToV2(report); case 2: + case 3: + return report; + default: + throw new Error(`Unknown report version: ${reportVersion}`); + } +}; + +const upgradeReport = (report) => { + const reportVersion = getReportVersion(report); + + switch (reportVersion) { + case 1: + return upgradeReportV2ToV3(upgradeReportV1ToV2(report)); + case 2: + return upgradeReportV2ToV3(report); + case 3: return report; default: throw new Error(`Unknown report version: ${reportVersion}`); @@ -283,7 +367,7 @@ const upgradeReport = (report) => { }; class Report { - constructor(path, { context, lmsInfo, overrideContext = false } = {}) { + constructor(path, { context, lmsInfo, overrideContext = false, upgradeToLatest = false } = {}) { let report; try { @@ -310,13 +394,17 @@ class Report { this._reportVersionOriginal = reportVersionOriginal; - if (reportVersionOriginal < latestReportVersion) { - report = upgradeReport(report); + if (reportVersionOriginal < 2) { + report = upgradeReportToV2(report); + + validateReport(report, `report (v${getReportVersion(report)})`); + } - const reportVersionUpgraded = getReportVersion(report); + if (upgradeToLatest && getReportVersion(report) < latestReportVersion) { + report = upgradeReport(report); - validateReport(report, `report (v${reportVersionUpgraded})`); - } else if (reportVersionOriginal > latestReportVersion) { + validateReport(report, `report (v${getReportVersion(report)})`); + } else if (getReportVersion(report) > latestReportVersion) { throw new Error(`Unsupported report version specified: ${reportVersionOriginal}`); } diff --git a/src/helpers/schema.cjs b/src/helpers/schema.cjs index 36aee338..b3411613 100644 --- a/src/helpers/schema.cjs +++ b/src/helpers/schema.cjs @@ -2,7 +2,7 @@ const Ajv = require('ajv/dist/2019'); const addFormats = require('ajv-formats'); const addErrors = require('ajv-errors'); -const latestReportSchema = require('../../schemas/report/v2.json'); +const latestReportSchema = require('../../schemas/report/v3.json'); const ajv = new Ajv({ verbose: true, strict: true, @@ -14,6 +14,7 @@ addFormats(ajv, ['date-time', 'uri', 'uuid']); ajv.addSchema(require('../../schemas/report-configuration/v1.json')); ajv.addSchema(require('../../schemas/report/v1.json')); +ajv.addSchema(require('../../schemas/report/v2.json')); ajv.addSchema(latestReportSchema); ajv.addSchema({ $schema: 'https://json-schema.org/draft/2019-09/schema', @@ -29,12 +30,21 @@ ajv.addSchema({ type: 'object', unevaluatedProperties: true }); +ajv.addSchema({ + $schema: 'https://json-schema.org/draft/2019-09/schema', + $id: '/test-reporting/schemas/report/v3/context/loose.json', + $ref: '/test-reporting/schemas/report/v3/context.json', + type: 'object', + unevaluatedProperties: true +}); const validateReportConfigurationV1Ajv = ajv.getSchema('/test-reporting/schemas/report-configuration/v1.json'); const validateReportV1ContextAjv = ajv.getSchema('/test-reporting/schemas/report/v1/context/loose.json'); const validateReportV2ContextAjv = ajv.getSchema('/test-reporting/schemas/report/v2/context/loose.json'); +const validateReportV3ContextAjv = ajv.getSchema('/test-reporting/schemas/report/v3/context/loose.json'); const validateReportV1Ajv = ajv.getSchema('/test-reporting/schemas/report/v1.json'); const validateReportV2Ajv = ajv.getSchema('/test-reporting/schemas/report/v2.json'); +const validateReportV3Ajv = ajv.getSchema('/test-reporting/schemas/report/v3.json'); const formatErrorAjv = ajv.errorsText; const { properties: latestReportSchemaProperties } = latestReportSchema; const { version: { const: latestReportVersion } } = latestReportSchemaProperties; @@ -45,8 +55,10 @@ module.exports = { validateReportConfigurationV1Ajv, validateReportV1ContextAjv, validateReportV2ContextAjv, + validateReportV3ContextAjv, validateReportV1Ajv, validateReportV2Ajv, + validateReportV3Ajv, latestReportVersion, latestSupportedBrowsers }; diff --git a/src/reporters/webdriverio.cjs b/src/reporters/webdriverio.cjs index 8c10c962..20da0bc8 100644 --- a/src/reporters/webdriverio.cjs +++ b/src/reporters/webdriverio.cjs @@ -18,6 +18,7 @@ class WebdriverIO extends WDIOReporter { this._baseReportPath = options.reportPath || './d2l-test-report.json'; this._reportConfigurationPath = options.reportConfigurationPath || './d2l-test-reporting.config.json'; + this._reportVersionLatest = options.reportVersionLatest || false; this._verbose = options.verbose || false; this._logger = logger; this._report = null; @@ -63,6 +64,7 @@ class WebdriverIO extends WDIOReporter { this._report = new ReportBuilder('webdriverio', this._logger, { reportPath: workerReportPath, reportConfigurationPath: this._reportConfigurationPath, + reportVersionLatest: this._reportVersionLatest, verbose: this._verbose }); console.log('[D2L Reporter] Initialized successfully'); diff --git a/test/integration/data/configs/mocha.cjs b/test/integration/data/configs/mocha.cjs index 09f5c98a..170f0975 100644 --- a/test/integration/data/configs/mocha.cjs +++ b/test/integration/data/configs/mocha.cjs @@ -4,6 +4,7 @@ module.exports = { reporter: 'src/reporters/mocha.cjs', reporterOptions: [ 'reportPath=./d2l-test-report-mocha.json', + 'reportVersionLatest=true', 'verbose=true' ] }; diff --git a/test/integration/data/configs/playwright.js b/test/integration/data/configs/playwright.js index 6277557a..9c366999 100644 --- a/test/integration/data/configs/playwright.js +++ b/test/integration/data/configs/playwright.js @@ -2,6 +2,7 @@ import { defineConfig, devices } from '@playwright/test'; const playwrightReporterOptions = { reportPath: './d2l-test-report-playwright.json', + reportVersionLatest: true, verbose: true }; diff --git a/test/integration/data/configs/web-test-runner.js b/test/integration/data/configs/web-test-runner.js index 5b119e86..dd4703e0 100644 --- a/test/integration/data/configs/web-test-runner.js +++ b/test/integration/data/configs/web-test-runner.js @@ -8,6 +8,7 @@ export default { defaultReporter(), reporter({ reportPath: './d2l-test-report-web-test-runner.json', + reportVersionLatest: true, verbose: true }) ], diff --git a/test/integration/data/configs/webdriverio.cjs b/test/integration/data/configs/webdriverio.cjs index 354a24fa..400b1336 100644 --- a/test/integration/data/configs/webdriverio.cjs +++ b/test/integration/data/configs/webdriverio.cjs @@ -25,6 +25,7 @@ exports.config = { [join(__dirname, '../../../../src/reporters/webdriverio.cjs'), { reportPath: './d2l-test-report-webdriverio.json', reportConfigurationPath: './d2l-test-reporting.config.json', + reportVersionLatest: true, verbose: true }] ], diff --git a/test/unit/report-builder.test.js b/test/unit/report-builder.test.js index 9aed7270..0bbee542 100644 --- a/test/unit/report-builder.test.js +++ b/test/unit/report-builder.test.js @@ -64,22 +64,6 @@ describe('report builder', () => { afterEach(() => sandbox.restore()); describe('constructor', () => { - it('sets version', () => { - const builder = new ReportBuilder('mocha', noopLogger, { reportWriter: () => { } }); - - expect(builder.data.version).to.eq(latestReportVersion); - }); - - it('produces loadable report', () => { - const builder = buildValidReport(); - - sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(builder)); - - const report = new Report('./test-report.json', { context: testContext }); - - expect(report.getVersion()).to.eq(latestReportVersion); - }); - it('throws with both output options', () => { expect(() => new ReportBuilder('mocha', noopLogger, { reportPath: './report.json', @@ -105,6 +89,57 @@ describe('report builder', () => { expect(builder.data.id).to.be.a.uuid('v4'); }); + + describe('v2', () => { + it('sets version to 2', () => { + const builder = new ReportBuilder('mocha', noopLogger, { reportWriter: () => { } }); + + expect(builder.data.version).to.eq(2); + }); + + it('produces loadable report', () => { + const builder = buildValidReport(); + + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(builder)); + + const report = new Report('./test-report.json', { context: testContext }); + + expect(report.getVersion()).to.eq(2); + }); + + it('does not set codeowners on details', () => { + const builder = buildValidReport(); + const json = builder.toJSON(); + + expect(json.details[0]).to.not.have.nested.property('github.codeowners'); + }); + }); + + describe('v3', () => { + it('sets version to latest', () => { + const builder = new ReportBuilder('mocha', noopLogger, { reportWriter: () => { }, reportVersionLatest: true }); + + expect(builder.data.version).to.eq(latestReportVersion); + }); + + it('produces loadable report', () => { + const builder = buildValidReport({ reportVersionLatest: true }); + + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(builder)); + + const report = new Report('./test-report.json', { context: testContext }); + + expect(report.getVersion()).to.eq(latestReportVersion); + }); + + it('sets codeowners on details', () => { + const builder = buildValidReport({ reportVersionLatest: true }); + const json = builder.toJSON(); + + expect(json.details[0]).to.have.nested.property('github.codeowners'); + expect(json.details[0].github.codeowners).to.be.an('array').that.is.not.empty; + }); + }); }); describe('summary', () => { diff --git a/test/unit/report.test.js b/test/unit/report.test.js index 38924664..0f7704ff 100644 --- a/test/unit/report.test.js +++ b/test/unit/report.test.js @@ -5,6 +5,7 @@ import fs from 'node:fs'; import { latestReportVersion } from '../../src/helpers/schema.cjs'; import { Report } from '../../src/helpers/report.cjs'; import { resolve } from 'node:path'; + const testReportPath = resolve('./test-report.json'); const testContext = { github: { @@ -131,6 +132,78 @@ const testReportV1PartialContext = { countFlaky: 1 } }; +const testReportV2Full = { + id: '00000000-0000-0000-0000-000000000000', + version: 2, + summary: { + ...testContext, + operatingSystem: 'linux', + framework: 'mocha', + started: testStarted, + status: 'passed', + duration: { + total: 23857 + }, + count: { + passed: 2, + failed: 0, + skipped: 1, + flaky: 1 + } + }, + details: testDetails +}; +const testReportV2FullOther = { + ...testReportV2Full, + summary: { + ...testReportV2Full.summary, + ...testContextOther + } +}; +const testReportV2NoContext = { + ...testReportV2Full, + summary: { + operatingSystem: 'linux', + framework: 'mocha', + started: testStarted, + duration: { + total: 23857 + }, + status: 'passed', + count: { + passed: 2, + failed: 0, + skipped: 1, + flaky: 1 + } + } +}; +const testReportV2PartialContext = { + ...testReportV2Full, + summary: { + github: { + organization: testContext.github.organization, + workflow: testContext.github.workflow + }, + git: { + branch: testContext.git.branch, + sha: testContext.git.sha + }, + operatingSystem: 'linux', + framework: 'mocha', + started: testStarted, + duration: { + total: 23857 + }, + status: 'passed', + count: { + passed: 2, + failed: 0, + skipped: 1, + flaky: 1 + } + } +}; const testReportLatestFull = { id: '00000000-0000-0000-0000-000000000000', version: latestReportVersion, @@ -220,7 +293,7 @@ describe('report', () => { let report; - const wrapper = () => report = (new Report(testReportPath)); + const wrapper = () => report = (new Report(testReportPath, { upgradeToLatest: true })); expect(wrapper).to.not.throw(); expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); @@ -236,7 +309,8 @@ describe('report', () => { const reportOptions = { context: testContextOther, - overrideContext: true + overrideContext: true, + upgradeToLatest: true }; let report; @@ -253,7 +327,10 @@ describe('report', () => { it('inject context if needed', () => { sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV1Full)); - const reportOptions = { context: testContextOther }; + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; let report; const wrapper = () => report = new Report(testReportPath, reportOptions); @@ -273,7 +350,8 @@ describe('report', () => { const reportOptions = { context: testContextOther, - overrideContext: true + overrideContext: true, + upgradeToLatest: true }; let report; @@ -290,7 +368,10 @@ describe('report', () => { it('inject context if needed', () => { sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV1NoContext)); - const reportOptions = { context: testContextOther }; + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; let report; const wrapper = () => report = new Report(testReportPath, reportOptions); @@ -310,7 +391,8 @@ describe('report', () => { const reportOptions = { context: testContextOther, - overrideContext: true + overrideContext: true, + upgradeToLatest: true }; let report; @@ -327,7 +409,10 @@ describe('report', () => { it('inject context if needed', () => { sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV1PartialContext)); - const reportOptions = { context: testContextOther }; + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; let report; const wrapper = () => report = new Report(testReportPath, reportOptions); @@ -343,6 +428,150 @@ describe('report', () => { }); }); + describe(`legacy (v2, upgrades to v${latestReportVersion})`, () => { + const testReportCurrentVersion = 2; + + describe('construction', () => { + it('don\'t override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2Full)); + + let report; + + const wrapper = () => report = (new Report(testReportPath, { upgradeToLatest: true })); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFull); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); + expect(report.getContext()).to.deep.equal(testContext); + }); + + describe('full report', () => { + it('override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2Full)); + + const reportOptions = { + context: testContextOther, + overrideContext: true, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + + it('inject context if needed', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2Full)); + + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFull); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); + expect(report.getContext()).to.deep.equal(testContext); + }); + }); + + describe('no context', () => { + it('override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2NoContext)); + + const reportOptions = { + context: testContextOther, + overrideContext: true, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + + it('inject context if needed', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2NoContext)); + + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + }); + + describe('partial context', () => { + it('override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2PartialContext)); + + const reportOptions = { + context: testContextOther, + overrideContext: true, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + + it('inject context if needed', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2PartialContext)); + + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + }); + }); + }); + describe(`latest (v${latestReportVersion}, no upgrade)`, () => { describe('construction', () => { it('don\'t override context', () => { @@ -356,7 +585,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFull); - expect(report.toJSON()).to.deep.not.equal(testReportV1Full); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); expect(report.getContext()).to.deep.equal(testContext); }); @@ -376,7 +605,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); @@ -392,7 +621,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFull); - expect(report.toJSON()).to.deep.not.equal(testReportV1Full); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); expect(report.getContext()).to.deep.equal(testContext); }); }); @@ -413,7 +642,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); @@ -429,7 +658,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); }); @@ -450,7 +679,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); @@ -466,7 +695,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); });