From f559ab184c34535a19f70e04f243503c376d5b95 Mon Sep 17 00:00:00 2001 From: NickK21 Date: Sat, 18 Jul 2026 18:24:47 -0700 Subject: [PATCH] Improve pytest parsing for beginner-friendly evaluation feedback --- executor.js | 140 +------------------------ package.json | 2 +- pytest-parser.js | 237 ++++++++++++++++++++++++++++++++++++++++++ pytest-parser.test.js | 229 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 468 insertions(+), 140 deletions(-) create mode 100644 pytest-parser.js create mode 100644 pytest-parser.test.js diff --git a/executor.js b/executor.js index f951a30..b57cebe 100644 --- a/executor.js +++ b/executor.js @@ -2,6 +2,7 @@ const { exec, spawn } = require('child_process'); const path = require('path'); const fs = require('fs').promises; const { v4: uuidv4 } = require('uuid'); +const { parsePytestOutput } = require('./pytest-parser'); /** * Ensures the datasets repo is cloned or updated in the unique directory. @@ -237,145 +238,6 @@ ${testCode} } } -/** - * Parses pytest output to extract structured test results. - * @param {string} stdout - The stdout from pytest. - * @param {string} [stderr=''] - The stderr from pytest. - * @param {number|null} [exitCode=null] - The pytest process exit code. - * @returns {object} - The test summary. - */ -function buildRawOutput(stdout = '', stderr = '') { - if (stdout && stderr) { - return `${stdout}\n${stderr}`; - } - return stdout || stderr || ''; -} - -function extractPytestSummary(output) { - const lines = output - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - - const summaryLine = [...lines].reverse().find((line) => ( - /^=+/.test(line) && - /=+$/.test(line) && - (/\bin [\d.]+s\b/.test(line) || /\bno tests ran\b/.test(line)) - )); - - if (!summaryLine) { - return ''; - } - - return summaryLine.replace(/^=+\s*/, '').replace(/\s*=+$/, ''); -} - -function extractPytestCount(summary, labelPattern) { - const match = summary.match(new RegExp(`(\\d+) ${labelPattern}\\b`)); - return match ? parseInt(match[1], 10) : 0; -} - -function extractPytestShortSummaryTarget(output, prefix) { - const line = output - .split(/\r?\n/) - .map((entry) => entry.trim()) - .find((entry) => entry.startsWith(`${prefix} `)); - - return line ? line.slice(prefix.length + 1).trim() : ''; -} - -function extractPytestErrorMessage(output, stderr = '') { - const combined = buildRawOutput(output, stderr); - const patterns = [ - /^E\s+([A-Za-z_.]+(?:Error|Exception): .+)$/m, - /^([A-Za-z_.]+(?:Error|Exception): .+)$/m, - /^(ImportError while importing test module .+)$/m, - /^E\s+(.+)$/m, - ]; - - for (const pattern of patterns) { - const match = combined.match(pattern); - if (match) { - return match[1].trim(); - } - } - - return 'Pytest error during collection or execution'; -} - -function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { - const summary = extractPytestSummary(stdout); - const rawout = buildRawOutput(stdout, stderr); - const passed_tests = extractPytestCount(summary, 'passed'); - const failed_tests = extractPytestCount(summary, 'failed'); - const errors = extractPytestCount(summary, 'error(?:s)?'); - const no_tests_collected = exitCode === 5 || /\bno tests ran\b/.test(summary); - const failures = []; - - const failureBlocks = stdout.split(/={10,} FAILURES ={10,}/)[1]?.split(/={10,}/)[0] || ''; - const matches = [...failureBlocks.matchAll( - /_{5,}\s*(.*?)\s*_{5,}[\s\S]*?>\s*assert\s+(.*?)\s*?\nE\s+assert\s+(.*?)\s*?(?:\nE\s+\+\s+where\s+(.*?)\s+=)?/g - )]; - - matches.forEach((match, index) => { - const test_case = match[1]?.trim() || `Test ${index + 1}`; - const assertionLine = match[2]?.trim(); - const failedExpr = match[3]?.trim(); - const evaluated = match[4]?.trim() || ''; - - failures.push({ - test_case, - expected: failedExpr.split('==')[1]?.trim() || '', - received: evaluated || failedExpr.split('==')[0]?.trim(), - error_message: `Assertion failed: ${assertionLine}`, - rawout, - }); - }); - - if (failed_tests > 0 && failures.length === 0) { - failures.push({ - test_case: extractPytestShortSummaryTarget(stdout, 'FAILED') || 'pytest assertion failure', - expected: '', - received: '', - error_message: 'Pytest reported one or more failed assertions', - rawout, - }); - } - - let runtime_error = ''; - - if (errors > 0) { - runtime_error = extractPytestErrorMessage(stdout, stderr); - failures.push({ - test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution', - expected: '', - received: '', - error_message: runtime_error, - rawout, - }); - } else if (no_tests_collected) { - runtime_error = 'Pytest did not collect any tests'; - failures.push({ - test_case: 'pytest collection', - expected: 'at least 1 collected test', - received: '0 collected tests', - error_message: runtime_error, - rawout, - }); - } - - return { - tests_run: passed_tests + failed_tests, - passed: passed_tests, - failed: failed_tests, - errors, - no_tests_collected, - exit_code: exitCode, - failure_details: failures, - runtime_error, - }; -} - function parseCppTestOutput(output, stdout = '', stderr = '') { output = output.toString(); let total_tests = 0; diff --git a/package.json b/package.json index 8625d81..38f42c8 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "node --test" }, "keywords": [], "author": "", diff --git a/pytest-parser.js b/pytest-parser.js new file mode 100644 index 0000000..6387c05 --- /dev/null +++ b/pytest-parser.js @@ -0,0 +1,237 @@ +function buildRawOutput(stdout = '', stderr = '') { + if (stdout && stderr) { + return `${stdout}\n${stderr}`; + } + + return stdout || stderr || ''; +} + +function extractPytestSummary(output) { + const lines = output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + + const summaryLine = [...lines].reverse().find((line) => ( + /^=+/.test(line) && + /=+$/.test(line) && + (/\bin [\d.]+s\b/.test(line) || /\bno tests ran\b/.test(line)) + )); + + if (!summaryLine) { + return ''; + } + + return summaryLine.replace(/^=+\s*/, '').replace(/\s*=+$/, ''); +} + +function extractPytestCount(summary, labelPattern) { + const match = summary.match(new RegExp(`(\\d+) ${labelPattern}\\b`)); + return match ? parseInt(match[1], 10) : 0; +} + +function normalizePytestTarget(target = '') { + const trimmed = target.trim(); + + if (!trimmed) { + return ''; + } + + const segments = trimmed.split('::').filter(Boolean); + + if (segments.length > 1) { + return segments[segments.length - 1].trim(); + } + + const pathSegments = trimmed.split(/[\\/]/).filter(Boolean); + return pathSegments[pathSegments.length - 1] || trimmed; +} + +function extractPytestShortSummaryTarget(output, prefix) { + const line = output + .split(/\r?\n/) + .map((entry) => entry.trim()) + .find((entry) => entry.startsWith(`${prefix} `)); + + return line ? normalizePytestTarget(line.slice(prefix.length + 1).trim()) : ''; +} + +function extractPytestErrorMessage(output, stderr = '') { + const combined = buildRawOutput(output, stderr); + const patterns = [ + /^\s*E\s+([A-Za-z_.]+(?:Error|Exception): .+)$/m, + /^([A-Za-z_.]+(?:Error|Exception): .+)$/m, + /^(ImportError while importing test module .+)$/m, + /^\s*E\s+(.+)$/m, + ]; + + for (const pattern of patterns) { + const match = combined.match(pattern); + if (match) { + return match[1].trim(); + } + } + + return 'Pytest error during collection or execution'; +} + +function extractPytestFailureSection(stdout = '') { + const failureSectionMatch = stdout.match( + /={10,}\s+FAILURES\s+={10,}\n([\s\S]*?)(?=\n={10,}\s+(?:short test summary info|ERRORS)\s+={10,}|\n={10,}\s+\d+ .+? in [\d.]+s\s+={10,}|$)/i + ); + + return failureSectionMatch ? failureSectionMatch[1] : ''; +} + +function extractPytestFailureBlocks(stdout = '') { + const failureSection = extractPytestFailureSection(stdout); + + if (!failureSection) { + return []; + } + + const blockRegex = /_{5,}\s*(.*?)\s*_{5,}\n([\s\S]*?)(?=\n_{5,}\s*.*?\s*_{5,}\n|$)/g; + const blocks = []; + + for (const match of failureSection.matchAll(blockRegex)) { + blocks.push({ + title: normalizePytestTarget(match[1] || ''), + body: match[2] || '', + }); + } + + return blocks; +} + +function extractPytestAssertionDetails(body = '') { + const sourceLineMatch = body.match(/^\s*>\s*(.+)$/m); + const errorLines = [...body.matchAll(/^\s*E\s+(.+)$/gm)] + .map((match) => match[1].trim()) + .filter((line) => line && !/^\+\s+where\b/.test(line)); + const technicalLine = errorLines[errorLines.length - 1] || ''; + + if (!technicalLine.startsWith('assert ')) { + return null; + } + + const expression = technicalLine.replace(/^assert\s+/, '').trim(); + const operators = [' is not ', ' is ', '==']; + const operator = operators.find((candidate) => expression.includes(candidate)); + + if (!operator) { + return null; + } + + const [receivedSide = '', expectedSide = ''] = expression.split(operator); + + return { + assertionLine: sourceLineMatch ? sourceLineMatch[1].trim() : '', + technicalLine, + expected: expectedSide.trim(), + received: receivedSide.trim(), + }; +} + +function extractPytestAssertionMessage(body = '') { + const errorLines = [...body.matchAll(/^\s*E\s+(.+)$/gm)] + .map((match) => match[1].trim()) + .filter((line) => line && !/^\+\s+where\b/.test(line)); + + return errorLines.find((line) => /^AssertionError:/i.test(line)) || ''; +} + +function extractPytestFailureDetail(block, rawout) { + const sourceLineMatch = block.body.match(/^\s*>\s*(.+)$/m); + const errorLines = [...block.body.matchAll(/^\s*E\s+(.+)$/gm)] + .map((match) => match[1].trim()) + .filter((line) => line && !/^\+\s+where\b/.test(line)); + const technicalLine = errorLines[errorLines.length - 1] || ''; + const assertionDetails = extractPytestAssertionDetails(block.body); + const assertionMessage = extractPytestAssertionMessage(block.body); + + if (assertionDetails) { + return { + test_case: block.title || 'pytest assertion failure', + expected: assertionDetails.expected, + received: assertionDetails.received, + error_message: assertionMessage || `Assertion failed: ${assertionDetails.assertionLine || assertionDetails.technicalLine}`, + rawout, + }; + } + + return { + test_case: block.title || 'pytest failure', + expected: '', + received: '', + error_message: assertionMessage || technicalLine || (sourceLineMatch ? `Assertion failed: ${sourceLineMatch[1].trim()}` : 'Pytest reported a failure'), + rawout, + }; +} + +function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { + const summary = extractPytestSummary(stdout); + const rawout = buildRawOutput(stdout, stderr); + const passed_tests = extractPytestCount(summary, 'passed'); + const failed_tests = extractPytestCount(summary, 'failed'); + const errors = extractPytestCount(summary, 'error(?:s)?'); + const no_tests_collected = exitCode === 5 || /\bno tests ran\b/.test(summary); + const failures = []; + const failureBlocks = extractPytestFailureBlocks(stdout); + + failureBlocks.forEach((block) => { + failures.push(extractPytestFailureDetail(block, rawout)); + }); + + if (failed_tests > 0 && failures.length === 0) { + failures.push({ + test_case: extractPytestShortSummaryTarget(stdout, 'FAILED') || 'pytest assertion failure', + expected: '', + received: '', + error_message: extractPytestErrorMessage(stdout, stderr), + rawout, + }); + } + + let runtime_error = ''; + + if (errors > 0) { + runtime_error = extractPytestErrorMessage(stdout, stderr); + failures.push({ + test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution', + expected: '', + received: '', + error_message: runtime_error, + rawout, + }); + } else if (no_tests_collected) { + runtime_error = 'Pytest did not collect any tests'; + failures.push({ + test_case: 'pytest collection', + expected: 'at least 1 collected test', + received: '0 collected tests', + error_message: runtime_error, + rawout, + }); + } + + return { + tests_run: passed_tests + failed_tests, + passed: passed_tests, + failed: failed_tests, + errors, + no_tests_collected, + exit_code: exitCode, + failure_details: failures, + runtime_error, + }; +} + +module.exports = { + buildRawOutput, + extractPytestSummary, + extractPytestCount, + extractPytestShortSummaryTarget, + extractPytestErrorMessage, + extractPytestFailureBlocks, + parsePytestOutput, +}; diff --git a/pytest-parser.test.js b/pytest-parser.test.js new file mode 100644 index 0000000..7f229b0 --- /dev/null +++ b/pytest-parser.test.js @@ -0,0 +1,229 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { parsePytestOutput, extractPytestShortSummaryTarget } = require('./pytest-parser'); + +test('extracts the real technical message for pytest attribute failures', () => { + const stdout = ` +============================= test session starts ============================== +platform darwin -- Python 3.9.10, pytest-7.4.0, pluggy-1.2.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.numberOfHats == 9 +E AttributeError: module 'program' has no attribute 'numberOfHats' + +/tmp/test_program.py:4: AttributeError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables +============================== 1 failed in 0.02s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failed, 1); + assert.equal(parsed.failure_details.length, 1); + assert.equal(parsed.failure_details[0].test_case, 'test_hat_variables'); + assert.equal( + parsed.failure_details[0].error_message, + "AttributeError: module 'program' has no attribute 'numberOfHats'" + ); + assert.match(parsed.failure_details[0].rawout, /FAILED test_program\.py::test_hat_variables/); +}); + +test('normalizes pytest short summary targets to the test name', () => { + const stdout = ` +FAILED ../../../../../../../var/folders/example/test_program.py::test_hat_variables + `.trim(); + + assert.equal( + extractPytestShortSummaryTarget(stdout, 'FAILED'), + 'test_hat_variables' + ); +}); + +test('preserves custom assertion messages for hasattr-style lesson failures', () => { + const stdout = ` +============================= test session starts ============================== +platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" +E AssertionError: There should be a variable named exactly HatName +E assert False +E + where False = hasattr(program, 'HatName') + +test_program.py:8: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - AssertionError: There should be ... +============================== 1 failed in 0.01s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failed, 1); + assert.equal(parsed.failure_details.length, 1); + assert.equal( + parsed.failure_details[0].error_message, + 'AssertionError: There should be a variable named exactly HatName' + ); + assert.equal(parsed.failure_details[0].expected, ''); + assert.equal(parsed.failure_details[0].received, ''); +}); + +test('preserves custom assertion messages for printed dataframe regression cases', () => { + const stdout = ` +============================= test session starts ============================== +platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +____________________________ test_describe_printed _____________________________ + + def test_describe_printed(capsys): +> assert part in out, f"Expected '{part}' in output, got:\\n{out}" +E AssertionError: Expected 'count' in output, got: +E 0 1 2 3 + +test_program.py:12: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_describe_printed - AssertionError: Expected 'cou... +============================== 1 failed in 0.01s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failed, 1); + assert.equal(parsed.failure_details.length, 1); + assert.equal(parsed.failure_details[0].test_case, 'test_describe_printed'); + assert.equal( + parsed.failure_details[0].error_message, + "AssertionError: Expected 'count' in output, got:" + ); +}); + +test('extracts expected and received values for numeric equality assertions', () => { + const stdout = ` +============================= test session starts ============================== +platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.NumberOfHats == 9 +E assert 8 == 9 + +test_program.py:12: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - assert 8 == 9 +============================== 1 failed in 0.01s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.NumberOfHats == 9'); + assert.equal(parsed.failure_details[0].received, '8'); + assert.equal(parsed.failure_details[0].expected, '9'); +}); + +test('extracts expected and received values for string equality assertions', () => { + const stdout = ` +============================= test session starts ============================== +platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.HatName == "Veracruz" +E assert "Oaxaca" == "Veracruz" + +test_program.py:10: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - assert "Oaxaca" == "Veracruz" +============================== 1 failed in 0.01s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.HatName == "Veracruz"'); + assert.equal(parsed.failure_details[0].received, '"Oaxaca"'); + assert.equal(parsed.failure_details[0].expected, '"Veracruz"'); +}); + +test('extracts expected and received values for float equality assertions', () => { + const stdout = ` +============================= test session starts ============================== +platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.CostOfHats == 278.91 +E assert 199.99 == 278.91 + +test_program.py:14: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - assert 199.99 == 278.91 +============================== 1 failed in 0.01s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.CostOfHats == 278.91'); + assert.equal(parsed.failure_details[0].received, '199.99'); + assert.equal(parsed.failure_details[0].expected, '278.91'); +}); + +test('extracts expected and received values for boolean identity assertions', () => { + const stdout = ` +============================= test session starts ============================== +platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0 +collected 1 item + +test_program.py F [100%] + +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.WearingHat is False +E assert True is False +E + where True = program.WearingHat + +test_program.py:16: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - assert True is False +============================== 1 failed in 0.01s =============================== + `.trim(); + + const parsed = parsePytestOutput(stdout, '', 1); + + assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.WearingHat is False'); + assert.equal(parsed.failure_details[0].received, 'True'); + assert.equal(parsed.failure_details[0].expected, 'False'); +});