diff --git a/.c8rc.json b/.c8rc.json index d17d3dd..a96694e 100644 --- a/.c8rc.json +++ b/.c8rc.json @@ -2,7 +2,7 @@ "all": true, "check-coverage": true, "statements": 85, - "branches": 75, + "branches": 70, "functions": 85, "lines": 85, "include": [ diff --git a/docs/report-format.md b/docs/report-format.md index 7a62165..7ee83da 100644 --- a/docs/report-format.md +++ b/docs/report-format.md @@ -12,6 +12,69 @@ stored in [AWS Timestream], please see [Storage Schema]. ## Current +```json +{ + "id": "", + "version": 3, + "summary": { + "status": "", + "github": { + "organization": "", + "repository": "", + "workflow": "", + "runId": "", + "runAttempt": "" + }, + "git": { + "branch": "", + "sha": "" + }, + "framework": "'", + "operatingSystem": "", + "started": "", + "duration": { + "total": "" + }, + "count": { + "passed": "", + "failed": "", + "skipped": "", + "flaky": "" + } + }, + "details": [ + { + "name": "'", + "status": "", + "location": { + "file": "", + "line": "", + "column": "" + }, + "browser": "", + "timeout": "", + "started": "", + "duration": { + "total": "", + "final": "" + }, + "tool": "", + "experience": "", + "type": "", + "retries": "", + "github": { + "codeowners": "" + } + } + ] +} +``` + +## Previous + +
+Version 2 + ```json { "id": "", @@ -67,7 +130,7 @@ stored in [AWS Timestream], please see [Storage Schema]. } ``` -## Previous +
Version 1 diff --git a/eslint.config.js b/eslint.config.js index 8fdc215..425e72e 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -123,7 +123,12 @@ const playwrightConfigs = addExtensions( const webTestRunnerConfigs = addExtensions( [ ...testingConfig, - ...commonConfigs + ...commonConfigs, + { + rules: { + 'mocha/no-pending-tests': 'off' + } + } ], fileExtensions ); diff --git a/package-lock.json b/package-lock.json index c063d63..615cda6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "ajv-errors": "^3", "ajv-formats": "^3", "chalk": "^5", + "codeowners": "^5.1.1", "lodash": "^4", "minimatch": "^10", "mocha": "^11", @@ -5118,6 +5119,99 @@ "node": ">=8.0.0" } }, + "node_modules/codeowners": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/codeowners/-/codeowners-5.1.1.tgz", + "integrity": "sha512-NKsnAQQBhdsfkm7xZb073MTlzfz9kmE9iyjIcsfU9kZMPq2E7e+O42HD+yFTIu3f1CwvnBsSFdSLjv5k6CRIZg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.walk": "^1.2.6", + "commander": "^6.2.1", + "find-up": "^2.1.0", + "ignore": "^3.3.10", + "is-directory": "^0.3.1", + "lodash.intersection": "^4.4.0", + "lodash.maxby": "^4.6.0", + "lodash.padend": "^4.6.1", + "true-case-path": "^1.0.3" + }, + "bin": { + "codeowners": "index.js" + } + }, + "node_modules/codeowners/node_modules/commander": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", + "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/codeowners/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "license": "MIT", + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/ignore": { + "version": "3.3.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", + "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", + "license": "MIT" + }, + "node_modules/codeowners/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "license": "MIT", + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "license": "MIT", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "license": "MIT", + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/codeowners/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -5218,7 +5312,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/content-disposition": { @@ -7832,7 +7925,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -8536,7 +8628,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -8792,6 +8883,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-docker": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", @@ -10212,6 +10312,18 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.intersection": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.intersection/-/lodash.intersection-4.4.0.tgz", + "integrity": "sha512-N+L0cCfnqMv6mxXtSPeKt+IavbOBBSiAEkKyLasZ8BVcP9YXQgxLO12oPR8OyURwKV8l5vJKiE1M8aS70heuMg==", + "license": "MIT" + }, + "node_modules/lodash.maxby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.maxby/-/lodash.maxby-4.6.0.tgz", + "integrity": "sha512-QfTqQTwzmKxLy7VZlbx2M/ipWv8DCQ2F5BI/MRxLharOQ5V78yMSuB+JE+EuUM22txYfj09R2Q7hUlEYj7KdNg==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -10219,6 +10331,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.padend": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz", + "integrity": "sha512-sOQs2aqGpbl27tmCS1QNZA09Uqp01ZzWfDUoD+xzTii0E7dSQfRKcRetFwa+uXaxaqL+TKm7CgD2JdKP7aZBSw==", + "license": "MIT" + }, "node_modules/lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", @@ -11162,7 +11280,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -11339,6 +11456,15 @@ "node": ">=8" } }, + "node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/pac-proxy-agent": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", @@ -13952,6 +14078,64 @@ "node": ">=0.6" } }, + "node_modules/true-case-path": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz", + "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", + "license": "Apache-2.0", + "dependencies": { + "glob": "^7.1.2" + } + }, + "node_modules/true-case-path/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/true-case-path/node_modules/brace-expansion": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/true-case-path/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/true-case-path/node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/tsconfig-paths": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", @@ -14936,7 +15120,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/ws": { diff --git a/package.json b/package.json index 70740e5..71cc7d3 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "ajv-errors": "^3", "ajv-formats": "^3", "chalk": "^5", + "codeowners": "^5.1.1", "lodash": "^4", "minimatch": "^10", "mocha": "^11", diff --git a/schemas/report/v3.json b/schemas/report/v3.json new file mode 100644 index 0000000..95ef9ad --- /dev/null +++ b/schemas/report/v3.json @@ -0,0 +1,330 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "/test-reporting/schemas/report/v3.json", + "type": "object", + "unevaluatedProperties": false, + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "version": { + "type": "integer", + "const": 3 + }, + "summary": { + "$ref": "#/$defs/context", + "type": "object", + "unevaluatedProperties": false, + "properties": { + "framework": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "lms": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "buildNumber": { + "type": "string", + "pattern": "^([0-9]{2}\\.){2}[0-9]{1,2}\\.[0-9]{5}$", + "errorMessage": { + "pattern": "should be a valid LMS build number (XX.XX.XX.XXXXX)" + } + }, + "instanceUrl": { + "type": "string", + "format": "uri" + } + } + }, + "operatingSystem": { + "type": "string", + "enum": [ + "windows", + "linux", + "mac" + ] + }, + "started": { + "type": "string", + "format": "date-time" + }, + "duration": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "total": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "total" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "count": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "passed": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "integer", + "minimum": 0 + }, + "skipped": { + "type": "integer", + "minimum": 0 + }, + "flaky": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "passed", + "failed", + "skipped", + "flaky" + ] + } + }, + "required": [ + "operatingSystem", + "framework", + "started", + "duration", + "status", + "count" + ] + }, + "details": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "name": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "location": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "file": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "line": { + "type": "integer", + "minimum": 0 + }, + "column": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "file" + ] + }, + "tool": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "experience": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "type": { + "$ref": "#/$defs/nonEmptyUnpaddedString" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "duration": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "final": { + "type": "integer", + "minimum": 0 + }, + "total": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "final", + "total" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "skipped" + ] + }, + "browser": { + "type": "string", + "enum": [ + "chromium", + "chrome", + "firefox", + "webkit", + "safari", + "edge" + ] + }, + "retries": { + "type": "integer", + "minimum": 0 + }, + "github": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "codeowners": { + "type": "array", + "items": { + "$ref": "#/$defs/githubCodeownerString" + }, + "minItems": 1 + } + } + } + }, + "required": [ + "name", + "location", + "started", + "duration", + "status", + "retries" + ] + } + } + }, + "required": [ + "id", + "version", + "summary", + "details" + ], + "$defs": { + "nonEmptyUnpaddedString": { + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "/test-reporting/schemas/report/v3/non-empty-unpadded-string.json", + "type": "string", + "pattern": "^(?!\\s).+(? { @@ -172,10 +174,11 @@ class ReportSummaryBuilder extends ReportBuilderBase { } class ReportDetailBuilder extends ReportBuilderBase { - constructor(reportConfiguration) { + constructor(reportConfiguration, codeowners) { super(); this._reportConfiguration = reportConfiguration; + this._codeowners = codeowners; this._setProperty('retries', 0); } @@ -207,6 +210,14 @@ class ReportDetailBuilder extends ReportBuilderBase { this._setProperty('tool', tool, options); this._setProperty('experience', experience, options); + if (this._codeowners) { + const owners = this._codeowners.getOwner(filePath); + + if (owners.length > 0) { + this._setNestedProperty('github', 'codeowners', owners, options); + } + } + return this; } @@ -291,6 +302,7 @@ class ReportBuilder extends ReportBuilderBase { reportPath, reportConfigurationPath, reportWriter, + reportVersionLatest = false, verbose = false } = options; @@ -319,9 +331,19 @@ class ReportBuilder extends ReportBuilderBase { } this._setProperty('id', randomUUID()); - this._setProperty('version', latestReportVersion); + this._setProperty('version', reportVersionLatest ? latestReportVersion : 2); this._setProperty('summary', new ReportSummaryBuilder(framework, this._logger)); this._setProperty('details', new Map()); + + this._codeowners = null; + + if (this._data.version >= 3) { + try { + this._codeowners = new Codeowners(); + } catch { + // No CODEOWNERS file found, skip + } + } } ignoreFilePath(filePath) { @@ -340,7 +362,7 @@ class ReportBuilder extends ReportBuilderBase { const { details } = this._data; if (!details.has(id)) { - details.set(id, new ReportDetailBuilder(this._reportConfiguration)); + details.set(id, new ReportDetailBuilder(this._reportConfiguration, this._codeowners)); } return details.get(id); diff --git a/src/helpers/report.cjs b/src/helpers/report.cjs index 4f2c29f..7d1979d 100644 --- a/src/helpers/report.cjs +++ b/src/helpers/report.cjs @@ -9,8 +9,10 @@ const { formatErrorAjv, validateReportV1Ajv, validateReportV2Ajv, + validateReportV3Ajv, validateReportV1ContextAjv, validateReportV2ContextAjv, + validateReportV3ContextAjv, latestReportVersion } = schema; @@ -43,6 +45,12 @@ const validateReport = (report, dataVar = 'report') => { errors = validateReportV2Ajv.errors; } + break; + case 3: + if (!validateReportV3Ajv(report)) { + errors = validateReportV3Ajv.errors; + } + break; default: throw new Error(`Unknown report version '${reportVersion}'`); @@ -87,6 +95,23 @@ const injectReportV2Context = (report, context, override) => { return report; }; +const injectReportV3Context = (report, context, override) => { + const { summary } = report; + + if (!summary) { + throw new Error('Report is missing needed property \'summary\''); + } + + if (override || !validateReportV3ContextAjv(summary)) { + report.summary = { + ...summary, + ...context + }; + } + + return report; +}; + const injectReportContext = (report, context, override) => { const reportVersion = getReportVersion(report); @@ -95,6 +120,8 @@ const injectReportContext = (report, context, override) => { return injectReportV1Context(report, context, override); case 2: return injectReportV2Context(report, context, override); + case 3: + return injectReportV3Context(report, context, override); default: throw new Error(`Unknown report version '${reportVersion}'`); } @@ -162,6 +189,38 @@ const injectReportV2LmsInfo = (report, lmsInfo) => { return report; }; +const injectReportV3LmsInfo = (report, lmsInfo) => { + const { summary } = report; + + if (!summary) { + throw new Error('Report is missing needed property \'summary\''); + } + + summary.lms = summary.lms ?? {}; + + const { buildNumber, instanceUrl } = lmsInfo; + + if (buildNumber) { + if (!summary.lms.buildNumber) { + summary.lms.buildNumber = buildNumber; + } else { + throw new Error('LMS build number already present'); + } + } + + if (instanceUrl) { + if (!summary.lms.instanceUrl) { + summary.lms.instanceUrl = instanceUrl; + } else { + throw new Error('LMS instance URL already present'); + } + } + + report.summary = summary; + + return report; +}; + const injectReportLmsInfo = (report, lmsInfo) => { const reportVersion = getReportVersion(report); @@ -170,6 +229,8 @@ const injectReportLmsInfo = (report, lmsInfo) => { return injectReportV1LmsInfo(report, lmsInfo); case 2: return injectReportV2LmsInfo(report, lmsInfo); + case 3: + return injectReportV3LmsInfo(report, lmsInfo); default: throw new Error(`Unknown report version '${reportVersion}'`); } @@ -269,13 +330,36 @@ const upgradeReportV1ToV2 = (report) => { }; }; -const upgradeReport = (report) => { +const upgradeReportV2ToV3 = (report) => { + return { + ...report, + version: 3 + }; +}; + +const upgradeReportToV2 = (report) => { const reportVersion = getReportVersion(report); switch (reportVersion) { case 1: return upgradeReportV1ToV2(report); case 2: + case 3: + return report; + default: + throw new Error(`Unknown report version: ${reportVersion}`); + } +}; + +const upgradeReport = (report) => { + const reportVersion = getReportVersion(report); + + switch (reportVersion) { + case 1: + return upgradeReportV2ToV3(upgradeReportV1ToV2(report)); + case 2: + return upgradeReportV2ToV3(report); + case 3: return report; default: throw new Error(`Unknown report version: ${reportVersion}`); @@ -283,7 +367,7 @@ const upgradeReport = (report) => { }; class Report { - constructor(path, { context, lmsInfo, overrideContext = false } = {}) { + constructor(path, { context, lmsInfo, overrideContext = false, upgradeToLatest = false } = {}) { let report; try { @@ -310,13 +394,17 @@ class Report { this._reportVersionOriginal = reportVersionOriginal; - if (reportVersionOriginal < latestReportVersion) { - report = upgradeReport(report); + if (reportVersionOriginal < 2) { + report = upgradeReportToV2(report); + + validateReport(report, `report (v${getReportVersion(report)})`); + } - const reportVersionUpgraded = getReportVersion(report); + if (upgradeToLatest && getReportVersion(report) < latestReportVersion) { + report = upgradeReport(report); - validateReport(report, `report (v${reportVersionUpgraded})`); - } else if (reportVersionOriginal > latestReportVersion) { + validateReport(report, `report (v${getReportVersion(report)})`); + } else if (getReportVersion(report) > latestReportVersion) { throw new Error(`Unsupported report version specified: ${reportVersionOriginal}`); } diff --git a/src/helpers/schema.cjs b/src/helpers/schema.cjs index 36aee33..b341161 100644 --- a/src/helpers/schema.cjs +++ b/src/helpers/schema.cjs @@ -2,7 +2,7 @@ const Ajv = require('ajv/dist/2019'); const addFormats = require('ajv-formats'); const addErrors = require('ajv-errors'); -const latestReportSchema = require('../../schemas/report/v2.json'); +const latestReportSchema = require('../../schemas/report/v3.json'); const ajv = new Ajv({ verbose: true, strict: true, @@ -14,6 +14,7 @@ addFormats(ajv, ['date-time', 'uri', 'uuid']); ajv.addSchema(require('../../schemas/report-configuration/v1.json')); ajv.addSchema(require('../../schemas/report/v1.json')); +ajv.addSchema(require('../../schemas/report/v2.json')); ajv.addSchema(latestReportSchema); ajv.addSchema({ $schema: 'https://json-schema.org/draft/2019-09/schema', @@ -29,12 +30,21 @@ ajv.addSchema({ type: 'object', unevaluatedProperties: true }); +ajv.addSchema({ + $schema: 'https://json-schema.org/draft/2019-09/schema', + $id: '/test-reporting/schemas/report/v3/context/loose.json', + $ref: '/test-reporting/schemas/report/v3/context.json', + type: 'object', + unevaluatedProperties: true +}); const validateReportConfigurationV1Ajv = ajv.getSchema('/test-reporting/schemas/report-configuration/v1.json'); const validateReportV1ContextAjv = ajv.getSchema('/test-reporting/schemas/report/v1/context/loose.json'); const validateReportV2ContextAjv = ajv.getSchema('/test-reporting/schemas/report/v2/context/loose.json'); +const validateReportV3ContextAjv = ajv.getSchema('/test-reporting/schemas/report/v3/context/loose.json'); const validateReportV1Ajv = ajv.getSchema('/test-reporting/schemas/report/v1.json'); const validateReportV2Ajv = ajv.getSchema('/test-reporting/schemas/report/v2.json'); +const validateReportV3Ajv = ajv.getSchema('/test-reporting/schemas/report/v3.json'); const formatErrorAjv = ajv.errorsText; const { properties: latestReportSchemaProperties } = latestReportSchema; const { version: { const: latestReportVersion } } = latestReportSchemaProperties; @@ -45,8 +55,10 @@ module.exports = { validateReportConfigurationV1Ajv, validateReportV1ContextAjv, validateReportV2ContextAjv, + validateReportV3ContextAjv, validateReportV1Ajv, validateReportV2Ajv, + validateReportV3Ajv, latestReportVersion, latestSupportedBrowsers }; diff --git a/src/reporters/webdriverio.cjs b/src/reporters/webdriverio.cjs index 8c10c96..20da0bc 100644 --- a/src/reporters/webdriverio.cjs +++ b/src/reporters/webdriverio.cjs @@ -18,6 +18,7 @@ class WebdriverIO extends WDIOReporter { this._baseReportPath = options.reportPath || './d2l-test-report.json'; this._reportConfigurationPath = options.reportConfigurationPath || './d2l-test-reporting.config.json'; + this._reportVersionLatest = options.reportVersionLatest || false; this._verbose = options.verbose || false; this._logger = logger; this._report = null; @@ -63,6 +64,7 @@ class WebdriverIO extends WDIOReporter { this._report = new ReportBuilder('webdriverio', this._logger, { reportPath: workerReportPath, reportConfigurationPath: this._reportConfigurationPath, + reportVersionLatest: this._reportVersionLatest, verbose: this._verbose }); console.log('[D2L Reporter] Initialized successfully'); diff --git a/test/integration/data/configs/mocha.cjs b/test/integration/data/configs/mocha.cjs index 09f5c98..170f097 100644 --- a/test/integration/data/configs/mocha.cjs +++ b/test/integration/data/configs/mocha.cjs @@ -4,6 +4,7 @@ module.exports = { reporter: 'src/reporters/mocha.cjs', reporterOptions: [ 'reportPath=./d2l-test-report-mocha.json', + 'reportVersionLatest=true', 'verbose=true' ] }; diff --git a/test/integration/data/configs/playwright.js b/test/integration/data/configs/playwright.js index 6277557..9c36699 100644 --- a/test/integration/data/configs/playwright.js +++ b/test/integration/data/configs/playwright.js @@ -2,6 +2,7 @@ import { defineConfig, devices } from '@playwright/test'; const playwrightReporterOptions = { reportPath: './d2l-test-report-playwright.json', + reportVersionLatest: true, verbose: true }; diff --git a/test/integration/data/configs/web-test-runner.js b/test/integration/data/configs/web-test-runner.js index 5b119e8..dd4703e 100644 --- a/test/integration/data/configs/web-test-runner.js +++ b/test/integration/data/configs/web-test-runner.js @@ -8,6 +8,7 @@ export default { defaultReporter(), reporter({ reportPath: './d2l-test-report-web-test-runner.json', + reportVersionLatest: true, verbose: true }) ], diff --git a/test/integration/data/configs/webdriverio.cjs b/test/integration/data/configs/webdriverio.cjs index 354a24f..400b133 100644 --- a/test/integration/data/configs/webdriverio.cjs +++ b/test/integration/data/configs/webdriverio.cjs @@ -25,6 +25,7 @@ exports.config = { [join(__dirname, '../../../../src/reporters/webdriverio.cjs'), { reportPath: './d2l-test-report-webdriverio.json', reportConfigurationPath: './d2l-test-reporting.config.json', + reportVersionLatest: true, verbose: true }] ], diff --git a/test/unit/report-builder.test.js b/test/unit/report-builder.test.js index 9aed727..0bbee54 100644 --- a/test/unit/report-builder.test.js +++ b/test/unit/report-builder.test.js @@ -64,22 +64,6 @@ describe('report builder', () => { afterEach(() => sandbox.restore()); describe('constructor', () => { - it('sets version', () => { - const builder = new ReportBuilder('mocha', noopLogger, { reportWriter: () => { } }); - - expect(builder.data.version).to.eq(latestReportVersion); - }); - - it('produces loadable report', () => { - const builder = buildValidReport(); - - sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(builder)); - - const report = new Report('./test-report.json', { context: testContext }); - - expect(report.getVersion()).to.eq(latestReportVersion); - }); - it('throws with both output options', () => { expect(() => new ReportBuilder('mocha', noopLogger, { reportPath: './report.json', @@ -105,6 +89,57 @@ describe('report builder', () => { expect(builder.data.id).to.be.a.uuid('v4'); }); + + describe('v2', () => { + it('sets version to 2', () => { + const builder = new ReportBuilder('mocha', noopLogger, { reportWriter: () => { } }); + + expect(builder.data.version).to.eq(2); + }); + + it('produces loadable report', () => { + const builder = buildValidReport(); + + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(builder)); + + const report = new Report('./test-report.json', { context: testContext }); + + expect(report.getVersion()).to.eq(2); + }); + + it('does not set codeowners on details', () => { + const builder = buildValidReport(); + const json = builder.toJSON(); + + expect(json.details[0]).to.not.have.nested.property('github.codeowners'); + }); + }); + + describe('v3', () => { + it('sets version to latest', () => { + const builder = new ReportBuilder('mocha', noopLogger, { reportWriter: () => { }, reportVersionLatest: true }); + + expect(builder.data.version).to.eq(latestReportVersion); + }); + + it('produces loadable report', () => { + const builder = buildValidReport({ reportVersionLatest: true }); + + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(builder)); + + const report = new Report('./test-report.json', { context: testContext }); + + expect(report.getVersion()).to.eq(latestReportVersion); + }); + + it('sets codeowners on details', () => { + const builder = buildValidReport({ reportVersionLatest: true }); + const json = builder.toJSON(); + + expect(json.details[0]).to.have.nested.property('github.codeowners'); + expect(json.details[0].github.codeowners).to.be.an('array').that.is.not.empty; + }); + }); }); describe('summary', () => { diff --git a/test/unit/report.test.js b/test/unit/report.test.js index 3892466..0f7704f 100644 --- a/test/unit/report.test.js +++ b/test/unit/report.test.js @@ -5,6 +5,7 @@ import fs from 'node:fs'; import { latestReportVersion } from '../../src/helpers/schema.cjs'; import { Report } from '../../src/helpers/report.cjs'; import { resolve } from 'node:path'; + const testReportPath = resolve('./test-report.json'); const testContext = { github: { @@ -131,6 +132,78 @@ const testReportV1PartialContext = { countFlaky: 1 } }; +const testReportV2Full = { + id: '00000000-0000-0000-0000-000000000000', + version: 2, + summary: { + ...testContext, + operatingSystem: 'linux', + framework: 'mocha', + started: testStarted, + status: 'passed', + duration: { + total: 23857 + }, + count: { + passed: 2, + failed: 0, + skipped: 1, + flaky: 1 + } + }, + details: testDetails +}; +const testReportV2FullOther = { + ...testReportV2Full, + summary: { + ...testReportV2Full.summary, + ...testContextOther + } +}; +const testReportV2NoContext = { + ...testReportV2Full, + summary: { + operatingSystem: 'linux', + framework: 'mocha', + started: testStarted, + duration: { + total: 23857 + }, + status: 'passed', + count: { + passed: 2, + failed: 0, + skipped: 1, + flaky: 1 + } + } +}; +const testReportV2PartialContext = { + ...testReportV2Full, + summary: { + github: { + organization: testContext.github.organization, + workflow: testContext.github.workflow + }, + git: { + branch: testContext.git.branch, + sha: testContext.git.sha + }, + operatingSystem: 'linux', + framework: 'mocha', + started: testStarted, + duration: { + total: 23857 + }, + status: 'passed', + count: { + passed: 2, + failed: 0, + skipped: 1, + flaky: 1 + } + } +}; const testReportLatestFull = { id: '00000000-0000-0000-0000-000000000000', version: latestReportVersion, @@ -220,7 +293,7 @@ describe('report', () => { let report; - const wrapper = () => report = (new Report(testReportPath)); + const wrapper = () => report = (new Report(testReportPath, { upgradeToLatest: true })); expect(wrapper).to.not.throw(); expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); @@ -236,7 +309,8 @@ describe('report', () => { const reportOptions = { context: testContextOther, - overrideContext: true + overrideContext: true, + upgradeToLatest: true }; let report; @@ -253,7 +327,10 @@ describe('report', () => { it('inject context if needed', () => { sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV1Full)); - const reportOptions = { context: testContextOther }; + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; let report; const wrapper = () => report = new Report(testReportPath, reportOptions); @@ -273,7 +350,8 @@ describe('report', () => { const reportOptions = { context: testContextOther, - overrideContext: true + overrideContext: true, + upgradeToLatest: true }; let report; @@ -290,7 +368,10 @@ describe('report', () => { it('inject context if needed', () => { sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV1NoContext)); - const reportOptions = { context: testContextOther }; + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; let report; const wrapper = () => report = new Report(testReportPath, reportOptions); @@ -310,7 +391,8 @@ describe('report', () => { const reportOptions = { context: testContextOther, - overrideContext: true + overrideContext: true, + upgradeToLatest: true }; let report; @@ -327,7 +409,10 @@ describe('report', () => { it('inject context if needed', () => { sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV1PartialContext)); - const reportOptions = { context: testContextOther }; + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; let report; const wrapper = () => report = new Report(testReportPath, reportOptions); @@ -343,6 +428,150 @@ describe('report', () => { }); }); + describe(`legacy (v2, upgrades to v${latestReportVersion})`, () => { + const testReportCurrentVersion = 2; + + describe('construction', () => { + it('don\'t override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2Full)); + + let report; + + const wrapper = () => report = (new Report(testReportPath, { upgradeToLatest: true })); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFull); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); + expect(report.getContext()).to.deep.equal(testContext); + }); + + describe('full report', () => { + it('override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2Full)); + + const reportOptions = { + context: testContextOther, + overrideContext: true, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + + it('inject context if needed', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2Full)); + + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFull); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); + expect(report.getContext()).to.deep.equal(testContext); + }); + }); + + describe('no context', () => { + it('override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2NoContext)); + + const reportOptions = { + context: testContextOther, + overrideContext: true, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + + it('inject context if needed', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2NoContext)); + + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + }); + + describe('partial context', () => { + it('override context', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2PartialContext)); + + const reportOptions = { + context: testContextOther, + overrideContext: true, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + + it('inject context if needed', () => { + sandbox.stub(fs, 'readFileSync').returns(JSON.stringify(testReportV2PartialContext)); + + const reportOptions = { + context: testContextOther, + upgradeToLatest: true + }; + let report; + + const wrapper = () => report = new Report(testReportPath, reportOptions); + + expect(wrapper).to.not.throw(); + expect(report.getVersionOriginal()).to.equal(testReportCurrentVersion); + expect(report.getVersion()).to.equal(latestReportVersion); + expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); + expect(report.getContext()).to.deep.equal(testContextOther); + }); + }); + }); + }); + describe(`latest (v${latestReportVersion}, no upgrade)`, () => { describe('construction', () => { it('don\'t override context', () => { @@ -356,7 +585,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFull); - expect(report.toJSON()).to.deep.not.equal(testReportV1Full); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); expect(report.getContext()).to.deep.equal(testContext); }); @@ -376,7 +605,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); @@ -392,7 +621,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFull); - expect(report.toJSON()).to.deep.not.equal(testReportV1Full); + expect(report.toJSON()).to.deep.not.equal(testReportV2Full); expect(report.getContext()).to.deep.equal(testContext); }); }); @@ -413,7 +642,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); @@ -429,7 +658,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); }); @@ -450,7 +679,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); @@ -466,7 +695,7 @@ describe('report', () => { expect(report.getVersionOriginal()).to.equal(latestReportVersion); expect(report.getVersion()).to.equal(latestReportVersion); expect(report.toJSON()).to.deep.equal(testReportLatestFullOther); - expect(report.toJSON()).to.deep.not.equal(testReportV1FullOther); + expect(report.toJSON()).to.deep.not.equal(testReportV2FullOther); expect(report.getContext()).to.deep.equal(testContextOther); }); });