diff --git a/codewit/api/src/controllers/attempt.ts b/codewit/api/src/controllers/attempt.ts index 21d511c..64947aa 100644 --- a/codewit/api/src/controllers/attempt.ts +++ b/codewit/api/src/controllers/attempt.ts @@ -4,6 +4,7 @@ import { UserExerciseCompletion } from '../models/userExerciseCompletion'; import { UserModuleCompletion } from '../models/userModuleCompletion'; import { AttemptWithEval } from '../typings/response.types'; import { EvaluationPayload, EvaluationResponse, executeCodeEvaluation } from '../utils/codeEvalService'; +import { addLearnerHintsToEvaluation } from '../utils/learnerHints'; import { Language as LanguageEnum } from '@codewit/language'; function getEvaluationError(response: EvaluationResponse): string { @@ -83,9 +84,14 @@ async function createAttempt( let evalResponse: EvaluationResponse | null = null; try { - const response = await executeCodeEvaluation(evaluationPayload, cookies); + const rawResponse = await executeCodeEvaluation(evaluationPayload, cookies); + const response = addLearnerHintsToEvaluation(rawResponse, { + referenceTest: exercise.referenceTest, + submittedCode: code, + topic: exercise.topic, + title: exercise.title, + }); evalResponse = response; - console.log('Code evaluation response:', response); const { tests_run, passed } = response; const evalError = getEvaluationError(response); @@ -95,7 +101,6 @@ async function createAttempt( if (tests_run > 0) { const completionPercentage = Math.round((passed / tests_run) * 100); attempt.completionPercentage = completionPercentage; - console.log(`Completion Percentage: ${completionPercentage}%`); // Update UserExerciseCompletion const completion = passed / tests_run; @@ -184,7 +189,8 @@ async function createAttempt( console.warn('Code evaluation returned a passed state without runnable tests:', response); } } catch (err) { - console.error('Code evaluation failed:', err.message); + const errorMessage = err instanceof Error ? err.message : String(err); + console.error('Code evaluation failed:', errorMessage); throw new Error('Code evaluation failed'); } diff --git a/codewit/api/src/models/attempt.ts b/codewit/api/src/models/attempt.ts index 5ea78ec..bd95575 100644 --- a/codewit/api/src/models/attempt.ts +++ b/codewit/api/src/models/attempt.ts @@ -12,6 +12,7 @@ import { Model, InferAttributes, InferCreationAttributes, + CreationOptional, DataTypes, Sequelize, NonAttribute, @@ -24,14 +25,14 @@ class Attempt extends Model< InferAttributes, InferCreationAttributes > { - declare uid: number; - declare timestamp: Date; + declare uid: CreationOptional; + declare timestamp: CreationOptional; declare exercise: NonAttribute; declare user: NonAttribute; - declare submissionNumber: number; + declare submissionNumber: CreationOptional; declare code: string; - declare completionPercentage: number; - declare error: string; + declare completionPercentage: CreationOptional; + declare error: CreationOptional; declare exerciseUid: number; declare userUid: number; diff --git a/codewit/api/src/utils/exerciseContract.spec.ts b/codewit/api/src/utils/exerciseContract.spec.ts new file mode 100644 index 0000000..d17db09 --- /dev/null +++ b/codewit/api/src/utils/exerciseContract.spec.ts @@ -0,0 +1,300 @@ +import { extractExerciseContract } from './exerciseContract'; + +describe('extractExerciseContract', () => { + it('detects variable expectations from lesson tests', () => { + const contract = extractExerciseContract( + ` +import program + +def test_hat_variables(): + assert program.numberOfHats == 9 + `.trim(), + 'variable', + 'Collecting Hats' + ); + + expect(contract.lessonFamily).toBe('variable'); + expect(contract.expectedVariables).toContain('numberOfHats'); + expect(contract.expectedFunctions).toHaveLength(0); + }); + + it('detects variable expectations from hasattr checks', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_hat_variables(): + sys.modules.pop("program", None) + import program + assert hasattr(program, "HatName") + `.trim(), + 'variable', + 'Collecting Hats' + ); + + expect(contract.lessonFamily).toBe('variable'); + expect(contract.expectedVariables).toContain('HatName'); + expect(contract.expectedFunctions).toHaveLength(0); + }); + + it('detects aliased program functions from live lesson tests', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_choose_clothes_function_variants(): + sys.modules.pop("program", None) + import program + f = program.chooes_clothes + assert f("weds") == {"shirt": "pink"} + `.trim(), + 'function', + 'Wardrobe rules' + ); + + expect(contract.lessonFamily).toBe('function'); + expect(contract.expectedFunctions).toContain('chooes_clothes'); + expect(contract.expectedVariables).not.toContain('chooes_clothes'); + }); + + it('detects dataframe operations and expected functions', () => { + const contract = extractExerciseContract( + ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_load_dataframe(): + expected = pd.read_csv('world_cup.csv') + result = program.loadWorldCupData() + assert_frame_equal(result, expected) + `.trim(), + 'load dataframe', + 'Load World cup data' + ); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.expectedFunctions).toContain('loadWorldCupData'); + expect(contract.dataframeOperations).toContain('load_dataframe'); + }); + + it('detects console io lessons from production-style tests', () => { + const contract = extractExerciseContract( + ` +from pytest import MonkeyPatch +import builtins +import sys + +def run_program_with(monkeypatch, inputs): + monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs)) + sys.modules.pop("program", None) + import program + +def test_song_request(monkeypatch, capsys): + user_input = iter(["Hello"]) + run_program_with(monkeypatch, user_input) + captured = capsys.readouterr().out + expected_output = "What song would you like to add\\n" + assert captured == expected_output + `.trim(), + 'console io', + 'Song Request' + ); + + expect(contract.lessonFamily).toBe('console_io'); + expect(contract.usesInput).toBe(true); + expect(contract.usesConsoleOutput).toBe(true); + }); + + [ + { + topic: 'describe dataframe', + title: 'Statistics of hair product data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_describe_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.describe() + result = program.describeHairProducts() + assert_frame_equal(result, expected) + `.trim(), + operation: 'describe_dataframe', + expectedFunction: 'describeHairProducts', + }, + { + topic: 'query dataframe', + title: 'Search hair product data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_query_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.query("Brand == 'Eco Style'") + result = program.searchHairProducts() + assert_frame_equal(result, expected) + `.trim(), + operation: 'query_dataframe', + expectedFunction: 'searchHairProducts', + }, + { + topic: 'melt dataframe', + title: 'Unpivot hair data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_melt_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = pd.melt(source, id_vars=['Brand']) + result = program.unpivotHairData() + assert_frame_equal(result, expected) + `.trim(), + operation: 'melt_dataframe', + expectedFunction: 'unpivotHairData', + }, + { + topic: 'pivot dataframe', + title: 'Pivot hair product data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_pivot_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.pivot_table(index='Brand', values='Price', aggfunc='mean') + result = program.pivotHairProducts() + assert_frame_equal(result, expected) + `.trim(), + operation: 'pivot_dataframe', + expectedFunction: 'pivotHairProducts', + }, + ].forEach(({ topic, title, referenceTest, operation, expectedFunction }) => { + it(`detects dataframe operation ${operation} for ${title}`, () => { + const contract = extractExerciseContract(referenceTest, topic, title); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.dataframeOperations).toContain(operation); + expect(contract.expectedFunctions).toContain(expectedFunction); + }); + }); + + describe('regression cases', () => { + it('detects lowercase snake_case variable expectations from player stat lessons', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_player_name(): + sys.modules.pop("program", None) + import program + assert program.player_name == "Coolminivan" + +def test_is_player_on(): + assert program.is_player_on is True + +def test_level_player(): + assert program.level_player == 64 + assert isinstance(program.level_player, int) + `.trim(), + 'variable', + 'Player Stats' + ); + + expect(contract.lessonFamily).toBe('variable'); + expect(contract.expectedVariables).toEqual( + expect.arrayContaining(['player_name', 'is_player_on', 'level_player']) + ); + }); + + it('detects printed-output lessons that do not use input', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_outputs_no_input(capsys): + sys.modules.pop("program", None) + import program + out = capsys.readouterr().out + expected = ( + "480\\n" + "285\\n" + "405\\n" + "2.7857142857142856\\n" + "9\\n" + "9\\n" + ) + assert out == expected + `.trim(), + 'math operation', + 'Road Trip Math' + ); + + expect(contract.lessonFamily).toBe('general'); + expect(contract.usesConsoleOutput).toBe(true); + expect(contract.usesInput).toBe(false); + }); + + it('detects printed dataframe lessons from topic metadata even without assert_frame_equal', () => { + const contract = extractExerciseContract( + ` +import importlib +import sys + +def test_describe_printed(capsys): + sys.modules.pop("program", None) + import program + importlib.reload(program) + out = capsys.readouterr().out.strip() + expected_parts = ["count", "mean", "std", "min", "max"] + for part in expected_parts: + assert part in out, f"Expected '{part}' in output, got:\\n{out}" + `.trim(), + 'describe dataframe', + 'Statistics of hair product data' + ); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.usesConsoleOutput).toBe(true); + expect(contract.dataframeOperations).toContain('describe_dataframe'); + }); + + it('does not treat program.py file references as an expected learner variable', () => { + const contract = extractExerciseContract( + ` +import importlib +import sys +import os +import pytest + +def test_program_output(capsys): + if not os.path.exists("./datasets/worldcup.csv"): + pytest.skip("Dataset not found; skipping test") + + sys.modules.pop("program", None) + import program + importlib.reload(program) + + out = capsys.readouterr().out.strip() + assert out, "No output printed from program.py" + assert "RUNNER UP" in out + `.trim(), + 'load dataframe', + 'World Cup Load Dataframe' + ); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.expectedVariables).not.toContain('py'); + }); + }); +}); diff --git a/codewit/api/src/utils/exerciseContract.ts b/codewit/api/src/utils/exerciseContract.ts new file mode 100644 index 0000000..91aa160 --- /dev/null +++ b/codewit/api/src/utils/exerciseContract.ts @@ -0,0 +1,154 @@ +type LessonFamily = + | 'variable' + | 'function' + | 'console_io' + | 'dataframe' + | 'general'; + +interface ExerciseContract { + topic: string; + title: string; + lessonFamily: LessonFamily; + expectedVariables: string[]; + expectedFunctions: string[]; + expectedImports: string[]; + usesConsoleOutput: boolean; + usesInput: boolean; + usesDataframe: boolean; + dataframeOperations: string[]; +} + +function unique(values: string[]): string[] { + return [...new Set(values.filter(Boolean))]; +} + +function stripQuotedStrings(input: string): string { + return input + .replace(/"""[\s\S]*?"""/g, ' ') + .replace(/'''[\s\S]*?'''/g, ' ') + .replace(/"([^"\\]|\\.)*"/g, ' ') + .replace(/'([^'\\]|\\.)*'/g, ' '); +} + +function collectMatches(input: string, regex: RegExp): string[] { + return unique([...input.matchAll(regex)].map((match) => match[1]?.trim() ?? '')); +} + +function normalizeTopic(topic?: string | null): string { + return (topic ?? '').trim().toLowerCase(); +} + +function detectDataframeOperations(referenceTest: string, normalizedTopic: string): string[] { + const operations = new Set(); + + if (/\bread_(csv|excel|json|parquet)\s*\(/i.test(referenceTest) || normalizedTopic.includes('load dataframe')) { + operations.add('load_dataframe'); + } + + if (/\.describe\s*\(/i.test(referenceTest) || normalizedTopic.includes('describe dataframe')) { + operations.add('describe_dataframe'); + } + + if (/\.query\s*\(/i.test(referenceTest) || normalizedTopic.includes('query dataframe')) { + operations.add('query_dataframe'); + } + + if (/(?:^|[^A-Za-z_])(?:pd\.)?melt\s*\(|\.melt\s*\(/i.test(referenceTest) || normalizedTopic.includes('melt dataframe')) { + operations.add('melt_dataframe'); + } + + if (/(?:pivot_table|\.pivot\s*\(|\.pivot_table\s*\()/i.test(referenceTest) || normalizedTopic.includes('pivot dataframe')) { + operations.add('pivot_dataframe'); + } + + return [...operations]; +} + +function detectLessonFamily(normalizedTopic: string, expectedFunctions: string[], usesConsoleOutput: boolean, usesDataframe: boolean): LessonFamily { + if (usesDataframe) { + return 'dataframe'; + } + + if (normalizedTopic === 'console io') { + return 'console_io'; + } + + if (normalizedTopic === 'function' || expectedFunctions.length > 0) { + return 'function'; + } + + if (normalizedTopic === 'variable') { + return 'variable'; + } + + return 'general'; +} + +function extractImportedIdentifiers(referenceTest: string): string[] { + const imported = collectMatches(referenceTest, /from\s+program\s+import\s+([^\n]+)/g); + const importedNames = imported.flatMap((group) => group.split(',')) + .map((part) => part.trim().replace(/\s+as\s+.+$/, '')) + .filter(Boolean); + + return unique(importedNames); +} + +function extractHasattrIdentifiers(referenceTest: string): string[] { + return collectMatches(referenceTest, /hasattr\s*\(\s*program\s*,\s*["']([A-Za-z_][A-Za-z0-9_]*)["']\s*\)/g); +} + +function extractAliasedProgramFunctions(referenceTest: string): string[] { + const aliases = [...referenceTest.matchAll(/([A-Za-z_][A-Za-z0-9_]*)\s*=\s*program\.([A-Za-z_][A-Za-z0-9_]*)\b/g)]; + const expectedFunctions = aliases + .filter((match) => { + const alias = match[1]; + return new RegExp(`\\b${alias}\\s*\\(`).test(referenceTest); + }) + .map((match) => match[2]?.trim() ?? ''); + + return unique(expectedFunctions); +} + +function extractExerciseContract( + referenceTest: string, + topic?: string | null, + title?: string | null +): ExerciseContract { + const structuralReferenceTest = stripQuotedStrings(referenceTest); + const normalizedTopic = normalizeTopic(topic); + const directProgramFunctionCalls = collectMatches(structuralReferenceTest, /program\.([A-Za-z_][A-Za-z0-9_]*)\s*\(/g); + const aliasedProgramFunctions = extractAliasedProgramFunctions(structuralReferenceTest); + const expectedFunctions = unique([...directProgramFunctionCalls, ...aliasedProgramFunctions]); + const importedIdentifiers = extractImportedIdentifiers(referenceTest); + const expectedVariables = unique([ + ...collectMatches(structuralReferenceTest, /program\.([A-Za-z_][A-Za-z0-9_]*)\b(?!\s*\()/g), + ...extractHasattrIdentifiers(referenceTest), + ]) + .filter((identifier) => !expectedFunctions.includes(identifier)); + const usesConsoleOutput = /(capsys|captured\.out|stdout|print\s*\()/i.test(referenceTest); + const usesInput = /\binput\s*\(|monkeypatch\.setattr.*input/i.test(referenceTest); + const dataframeOperations = detectDataframeOperations(referenceTest, normalizedTopic); + const usesDataframe = dataframeOperations.length > 0 || /pandas|DataFrame|assert_frame_equal|Series/i.test(referenceTest); + + return { + topic: topic ?? '', + title: title ?? '', + lessonFamily: detectLessonFamily(normalizedTopic, expectedFunctions, usesConsoleOutput, usesDataframe), + expectedVariables: unique([...expectedVariables, ...importedIdentifiers.filter((identifier) => !expectedFunctions.includes(identifier))]), + expectedFunctions: unique([...expectedFunctions, ...importedIdentifiers.filter((identifier) => new RegExp(`\\b${identifier}\\s*\\(`).test(referenceTest))]), + expectedImports: importedIdentifiers, + usesConsoleOutput, + usesInput, + usesDataframe, + dataframeOperations, + }; +} + +export type { + ExerciseContract, + LessonFamily, +}; + +export { + extractExerciseContract, +}; diff --git a/codewit/api/src/utils/learnerHints.spec.ts b/codewit/api/src/utils/learnerHints.spec.ts new file mode 100644 index 0000000..1be3988 --- /dev/null +++ b/codewit/api/src/utils/learnerHints.spec.ts @@ -0,0 +1,1183 @@ +import type { EvaluationResponse } from './codeEvalService'; +import { addLearnerHintsToEvaluation } from './learnerHints'; + +const collectingHatsReferenceTest = ` +import sys + +def test_hat_variables(): + sys.modules.pop("program", None) + import program + assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" + assert program.HatName == "Veracruz" + assert hasattr(program,"NumberOfHats"), "There should be a variable named exactly NumberOfHats" + assert program.NumberOfHats == 9 + assert hasattr(program,"CostOfHats"), "There should be a variable named exactly CostOfHats" + assert program.CostOfHats == 278.91 + assert hasattr(program,"WearingHat"), "There should be a variable named exactly WearingHat" + assert program.WearingHat is False +`.trim(); + +describe('addLearnerHintsToEvaluation', () => { + it('builds a high-confidence variable mismatch hint from the lesson test', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'numberOfHats'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import program + +def test_hat_variables(): + assert program.numberOfHats == 9 + `.trim(), + submittedCode: 'NumberOfHats = int(9)', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('numberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('NumberOfHats'); + expect(hinted.learner_hint?.title).toContain('variable name'); + }); + + it('builds a dataframe hint for dataframe assertion mismatches', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_pivot_result', + expected: 'expected frame', + received: 'actual frame', + error_message: 'Assertion failed: assert_frame_equal(result, expected)', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_pivot_result(): + expected = pd.DataFrame({'a': [1]}) + result = program.buildPivotTable() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'def buildPivotTable():\n return df', + topic: 'pivot dataframe', + title: 'Pivot the standings table', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('DataFrame'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('pivot'); + }); + + it('builds a variable name hint from assertion-style lesson messages', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: 'AssertionError: There should be a variable named exactly HatName', + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + +> assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" +E AssertionError: There should be a variable named exactly HatName + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_hat_variables(): + sys.modules.pop("program", None) + import program + assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" + `.trim(), + submittedCode: 'hatName = "Veracruz"', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('variable name'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('HatName'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('hatName'); + }); + + it('builds a format-based variable name hint when underscores are added', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'NumberOfHats'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import program + +def test_hat_variables(): + assert program.NumberOfHats == 9 + `.trim(), + submittedCode: 'Number_Of_Hats = 9', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('NumberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('Number_Of_Hats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('underscores'); + }); + + it('builds a capitalization hint for a different expected variable name', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'CostOfHats'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'costOfHats = float(278.91)', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('CostOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('costOfHats'); + }); + + it('builds a missing variable hint when the name is absent entirely', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'WearingHat'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import program + +def test_hat_variables(): + assert program.WearingHat is False + `.trim(), + submittedCode: 'HatName = "Veracruz"\nNumberOfHats = 9\nCostOfHats = 278.91', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_variable'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('WearingHat'); + }); + + [ + { + identifier: 'HatName', + expected: '"Veracruz"', + received: '"Oaxaca"', + errorMessage: 'Assertion failed: assert program.HatName == "Veracruz"', + submittedCode: 'HatName = str("Oaxaca")\nNumberOfHats = int(9)\nCostOfHats = float(278.91)\nWearingHat = bool(False)', + }, + { + identifier: 'NumberOfHats', + expected: '9', + received: '8', + errorMessage: 'Assertion failed: assert program.NumberOfHats == 9', + submittedCode: 'HatName = str("Veracruz")\nNumberOfHats = int(8)\nCostOfHats = float(278.91)\nWearingHat = bool(False)', + }, + { + identifier: 'CostOfHats', + expected: '278.91', + received: '199.99', + errorMessage: 'Assertion failed: assert program.CostOfHats == 278.91', + submittedCode: 'HatName = str("Veracruz")\nNumberOfHats = int(9)\nCostOfHats = float(199.99)\nWearingHat = bool(False)', + }, + { + identifier: 'WearingHat', + expected: 'False', + received: 'True', + errorMessage: 'Assertion failed: assert program.WearingHat is False', + submittedCode: 'HatName = str("Veracruz")\nNumberOfHats = int(9)\nCostOfHats = float(278.91)\nWearingHat = bool(True)', + }, + ].forEach(({ identifier, expected, received, errorMessage, submittedCode }) => { + it(`builds a specific variable-value hint when ${identifier} has the wrong asserted value`, () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected, + received, + error_message: errorMessage, + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode, + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain(identifier); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(`\`${identifier}\``); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(`\`${received}\``); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(`\`${expected}\``); + }); + }); + + it('builds a variable-specific value hint from raw pytest assertion text when expected and received are missing', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AssertionError: assert 'WRONG' == 'Veracruz'", + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + +> assert program.HatName == "Veracruz" +E AssertionError: assert 'WRONG' == 'Veracruz' + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'HatName = "WRONG"', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('HatName'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`HatName`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain("`'WRONG'`"); + expect(hinted.failure_details[0].learner_hint?.summary).toContain("`'Veracruz'`"); + }); + + it('prefers raw pytest assertion values when structured comparison fields are noisy', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '======================= 1 failed, 3 passed in 0.01s ==========================', + received: 'True is False', + error_message: 'AssertionError: assert True is False', + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + +> assert program.WearingHat is False +E AssertionError: assert True is False + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'HatName = "Veracruz"\nNumberOfHats = 9\nCostOfHats = 278.91\nWearingHat = True', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('WearingHat'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`True`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`False`'); + expect(hinted.failure_details[0].learner_hint?.summary).not.toContain('1 failed, 3 passed'); + }); + + it('builds the value hint for the failing variable in multi-assert traceback output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: 'AssertionError: assert 0 == 9', + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): + assert program.HatName == "Veracruz" +> assert program.NumberOfHats == 9 +E AssertionError: assert 0 == 9 +E + where 0 = .NumberOfHats + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'HatName = "Veracruz"\nNumberOfHats = 0', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('NumberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`NumberOfHats`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`0`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`9`'); + }); + + it('builds a generic output mismatch hint from pytest diff-style assertion output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_funkos_list_operations', + expected: '', + received: '', + error_message: '+ WRONG OUTPUT', + rawout: ` +=================================== FAILURES =================================== +_________________________ test_funkos_list_operations __________________________ + +> assert out == expected +E assert "actual" == "expected" +E + WRONG OUTPUT + +test_program.py:15: AssertionError + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_funkos_list_operations(capsys): + sys.modules.pop("program", None) + import program + out = capsys.readouterr().out + expected = "expected" + assert out == expected + `.trim(), + submittedCode: 'print("actual")', + topic: 'array list', + title: 'Funkos', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + }); + + it('builds a function hint for aliased program function checks', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_choose_clothes_function_variants', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'chooes_clothes'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_choose_clothes_function_variants(): + sys.modules.pop("program", None) + import program + f = program.chooes_clothes + assert f("weds") == {'shoes': 'pink'} + `.trim(), + submittedCode: 'shirt = "pink"', + topic: 'function', + title: 'Wardrobe rules', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_function'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('function'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('chooes_clothes'); + }); + + it('builds an output mismatch hint for console input/output lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_song_request', + expected: 'What song would you like to add\\nI heard that Hello is a good song\\n', + received: 'Hello\\n', + error_message: 'Assertion failed: assert captured == expected_output', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +from pytest import MonkeyPatch +import builtins +import sys + +def run_program_with(monkeypatch, inputs): + monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs)) + sys.modules.pop("program", None) + import program + +def test_song_request(monkeypatch, capsys): + user_input = iter(["Hello"]) + run_program_with(monkeypatch, user_input) + captured = capsys.readouterr().out + expected_output = ( + "What song would you like to add \\n" + "I heard that Hello is a good song\\n" + ) + assert captured == expected_output + `.trim(), + submittedCode: 'song = input()\\nprint(song)', + topic: 'console io', + title: 'Song Request', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + expect(hinted.failure_details[0].learner_hint?.next_steps.join(' ')).toContain('line breaks'); + }); + + it('builds a generic output mismatch hint for boolean expression lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_points_low_branch', + expected: 'Keep practicing\n', + received: 'All-star\n', + error_message: 'Assertion failed: assert captured == expected_output', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +from pytest import MonkeyPatch +import builtins +import sys + +def run_program_with(monkeypatch, inputs): + monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs)) + sys.modules.pop("program", None) + import program + +def test_points_low_branch(monkeypatch, capsys): + user_input = iter(["8"]) + run_program_with(monkeypatch, user_input) + captured = capsys.readouterr().out + expected_output = "Keep practicing\\n" + assert captured == expected_output + `.trim(), + submittedCode: 'points = int(input())\nprint("All-star")', + topic: 'boolean expression', + title: 'Basketball', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + }); + + it('builds a generic output mismatch hint for for-loop lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_budgeting_total', + expected: '[50, 25, 15]', + received: '[50, 25]', + error_message: 'Assertion failed: assert result == expected', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys +sys.modules.pop("program", None) +import program + +def test_budgeting_total(): + result = program.build_budget_list() + expected = [50, 25, 15] + assert result == expected + `.trim(), + submittedCode: 'def build_budget_list():\n return [50, 25]', + topic: 'for loop', + title: 'Budgeting', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('final value'); + }); + + [ + { + title: 'Load hair product data', + topic: 'load dataframe', + summarySnippet: 'loading the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_load_dataframe(): + expected = pd.read_csv('./datasets/hair.csv') + result = program.loadHairProductData() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef loadHairProductData():\n return pd.DataFrame()', + }, + { + title: 'Statistics of hair product data', + topic: 'describe dataframe', + summarySnippet: 'describing the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_describe_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.describe() + result = program.describeHairProducts() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef describeHairProducts():\n return pd.DataFrame()', + }, + { + title: 'Search hair product data', + topic: 'query dataframe', + summarySnippet: 'querying the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_query_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.query(\"Brand == 'Eco Style'\") + result = program.searchHairProducts() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef searchHairProducts():\n return pd.DataFrame()', + }, + { + title: 'Unpivot hair data', + topic: 'melt dataframe', + summarySnippet: 'melting the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_melt_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = pd.melt(source, id_vars=['Brand']) + result = program.unpivotHairData() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef unpivotHairData():\n return pd.DataFrame()', + }, + { + title: 'Pivot hair product data', + topic: 'pivot dataframe', + summarySnippet: 'pivoting the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_pivot_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.pivot_table(index='Brand', values='Price', aggfunc='mean') + result = program.pivotHairProducts() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef pivotHairProducts():\n return pd.DataFrame()', + }, + ].forEach(({ title, topic, summarySnippet, referenceTest, submittedCode }) => { + it(`builds a dataframe mismatch hint for ${title}`, () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_dataframe_result', + expected: 'expected frame', + received: 'actual frame', + error_message: 'Assertion failed: assert_frame_equal(result, expected)', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest, + submittedCode, + topic, + title, + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(summarySnippet); + }); + }); + + describe('regression cases', () => { + it('builds a variable name hint for lowercase snake_case lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_player_name', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'player_name'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_player_name(): + sys.modules.pop("program", None) + import program + assert program.player_name == "Coolminivan" + +def test_is_player_on(): + assert program.is_player_on is True + `.trim(), + submittedCode: 'playerName = "Coolminivan"\nis_player_on = True\nlevel_player = int(64)\nhealth_player = 2.5', + topic: 'variable', + title: 'Player Stats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('player_name'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('playerName'); + }); + + it('builds an output mismatch hint for no-input print lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_outputs_no_input', + expected: '480\n285\n405\n2.7857142857142856\n9\n9\n', + received: '480\n285\n405\n2.7857142857142856\n9\n8\n', + error_message: 'Assertion failed: assert out == expected', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_outputs_no_input(capsys): + sys.modules.pop("program", None) + import program + out = capsys.readouterr().out + expected = ( + "480\\n" + "285\\n" + "405\\n" + "2.7857142857142856\\n" + "9\\n" + "9\\n" + ) + assert out == expected + `.trim(), + submittedCode: ` +gallons = 12 +avg_miles = 40 +distance_home = 195 +speed = 70 +total_miles = gallons * avg_miles +print(total_miles) +print(total_miles - distance_home) +print((total_miles - distance_home) + (3 * 40)) +print(195 / 70) +print(3 ** 2) +print(pow(2, 3)) + `.trim(), + topic: 'math operation', + title: 'Road Trip Math', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + }); + + it('builds a dataframe hint for printed dataframe lessons without explicit expected and received values', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_describe_printed', + expected: '', + received: '', + error_message: "AssertionError: Expected 'count' in output, got:", + rawout: "Assertion failed: assert 'count' in out", + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import importlib +import sys + +def test_describe_printed(capsys): + sys.modules.pop("program", None) + import program + importlib.reload(program) + out = capsys.readouterr().out.strip() + expected_parts = ["count", "mean", "std", "min", "max"] + for part in expected_parts: + assert part in out, f"Expected '{part}' in output, got:\\n{out}" + `.trim(), + submittedCode: ` +import pandas as pd + +hair_products = pd.read_csv("../datasets/hair.csv") +print(hair_products["Y_2019"].head()) + `.trim(), + topic: 'describe dataframe', + title: 'Statistics of hair product data', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('describing the DataFrame'); + }); + + it('builds a dataframe mismatch hint when a dataframe lesson returns the wrong object type', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_world_cup_query', + expected: '', + received: '', + error_message: "AttributeError: 'str' object has no attribute 'columns'", + rawout: "AttributeError: 'str' object has no attribute 'columns'", + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_world_cup_query(): + expected = pd.DataFrame({'CHAMPION': ['Italy']}) + assert_frame_equal(program.WC, expected) + `.trim(), + submittedCode: 'WC = "wrong"', + topic: 'query dataframe', + title: 'World Cup Query', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('DataFrame'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('querying the DataFrame'); + }); + + it('builds a dataframe mismatch hint when pandas query evaluation raises a name error', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_query_players', + expected: '', + received: '', + error_message: "name 'BACKTICK_QUOTED_STRING_Top_Player' is not defined", + rawout: ` +======================= ERROR collecting test_program.py ======================= +/usr/lib/python3.11/site-packages/pandas/core/computation/scope.py:232: in resolve + return self.temps[key] +E KeyError: 'BACKTICK_QUOTED_STRING_Top_Player' + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +import program + +def test_query_players(): + result = program.df.query('\`Top Player\` == "A.Judge" or \`Top Player\` == "B.Ruth"') + assert isinstance(result, pd.DataFrame) + `.trim(), + submittedCode: 'df = {"Top Player": "wrong"}', + topic: 'query dataframe', + title: 'Search Baseball data', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('querying the DataFrame'); + }); + + it('builds a dataframe mismatch hint when dataframe columns are missing', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_worldcup_describe', + expected: '', + received: '', + error_message: `KeyError: "None of [Index(['TEAMS', 'MATCHES PLAYED', 'GOALS SCORED'], dtype='str')] are in the [columns]"`, + rawout: ` +_______________________ ERROR collecting test_program.py _______________________ +test_program.py:2: in + desc = program.world_cup[["TEAMS", "MATCHES PLAYED", "GOALS SCORED"]] + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +import program + +def test_worldcup_describe(): + desc = program.world_cup[["TEAMS", "MATCHES PLAYED", "GOALS SCORED"]] + assert isinstance(desc, pd.DataFrame) + `.trim(), + submittedCode: 'world_cup = pd.DataFrame({"WRONG": [0]})', + topic: 'describe dataframe', + title: 'Statistics of World cup data', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('describing the DataFrame'); + }); + }); + + it('builds a syntax hint from traceback text when the lesson only returns raw output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'pytest collection', + expected: '', + received: '', + error_message: '', + rawout: 'SyntaxError: invalid syntax', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'def test_program():\n pass', + submittedCode: 'print("hello"', + topic: 'console io', + title: 'Song Request', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('syntax_error'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('invalid syntax'); + }); +}); diff --git a/codewit/api/src/utils/learnerHints.ts b/codewit/api/src/utils/learnerHints.ts new file mode 100644 index 0000000..cc61529 --- /dev/null +++ b/codewit/api/src/utils/learnerHints.ts @@ -0,0 +1,611 @@ +import type { EvaluationResponse } from './codeEvalService'; +import type { FailureDetail, LearnerHint } from '@codewit/interfaces'; +import { extractExerciseContract } from './exerciseContract'; + +interface LearnerHintContext { + referenceTest: string; + submittedCode: string; + topic?: string | null; + title?: string | null; +} + +type MatchReason = 'case' | 'format'; + +function createHint( + kind: LearnerHint['kind'], + confidence: LearnerHint['confidence'], + title: string, + summary: string, + next_steps: string[] +): LearnerHint { + return { + kind, + confidence, + title, + summary, + next_steps, + }; +} + +function buildDiagnosticText(detail: FailureDetail): string { + return [detail.error_message, detail.rawout] + .filter((value) => typeof value === 'string' && value.trim().length > 0) + .join('\n'); +} + +function extractFirstMatchingLine(input: string, pattern: RegExp): string { + const match = input.match(pattern); + return match ? match[0].trim() : ''; +} + +function normalizeInlineValue(value: string): string { + return value + .replace(/\r\n/g, '\n') + .replace(/\s+/g, ' ') + .trim(); +} + +function isSuspiciousComparisonValue(value: string): boolean { + const normalized = normalizeInlineValue(value); + + if (!normalized || normalized === '...') { + return true; + } + + return /={3,}|test session starts|short test summary|collected \d+ items|failed in \d|passed in \d|rootdir:/i.test( + normalized + ); +} + +function extractIdentifiers(code: string): string[] { + const reserved = new Set([ + 'False', 'None', 'True', 'and', 'as', 'assert', 'break', 'class', 'continue', + 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', + 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', + 'raise', 'return', 'try', 'while', 'with', 'yield', 'int', 'float', 'str', + 'list', 'dict', 'set', 'tuple', 'print', 'input' + ]); + + return [...new Set( + [...code.matchAll(/\b([A-Za-z_][A-Za-z0-9_]*)\b/g)] + .map((match) => match[1]) + .filter((identifier) => !reserved.has(identifier)) + )]; +} + +function normalizeIdentifier(identifier: string): string { + return identifier.replace(/_/g, '').toLowerCase(); +} + +function findIdentifierMatch(expectedIdentifier: string, submittedCode: string): null | { actual: string; reason: MatchReason } { + const identifiers = extractIdentifiers(submittedCode); + const caseMatch = identifiers.find((identifier) => ( + identifier !== expectedIdentifier && + identifier.toLowerCase() === expectedIdentifier.toLowerCase() + )); + + if (caseMatch) { + return { actual: caseMatch, reason: 'case' }; + } + + const formatMatch = identifiers.find((identifier) => ( + identifier !== expectedIdentifier && + normalizeIdentifier(identifier) === normalizeIdentifier(expectedIdentifier) + )); + + if (formatMatch) { + return { actual: formatMatch, reason: 'format' }; + } + + return null; +} + +function describeDataframeStep(topic: string, operations: string[]): string { + const normalizedTopic = topic.trim().toLowerCase(); + + if (normalizedTopic.includes('pivot dataframe') || operations.includes('pivot_dataframe')) { + return 'pivoting the DataFrame'; + } + + if (normalizedTopic.includes('melt dataframe') || operations.includes('melt_dataframe')) { + return 'melting the DataFrame'; + } + + if (normalizedTopic.includes('query dataframe') || operations.includes('query_dataframe')) { + return 'querying the DataFrame'; + } + + if (normalizedTopic.includes('describe dataframe') || operations.includes('describe_dataframe')) { + return 'describing the DataFrame'; + } + + if (normalizedTopic.includes('load dataframe') || operations.includes('load_dataframe')) { + return 'loading the DataFrame'; + } + + return 'working with the DataFrame'; +} + +function buildMissingIdentifierHint( + expectedIdentifier: string, + submittedCode: string, + lessonLabel: string, + isFunction: boolean +): LearnerHint { + const similar = findIdentifierMatch(expectedIdentifier, submittedCode); + + if (similar?.reason === 'case') { + return createHint( + 'name_mismatch', + 'high', + `The ${isFunction ? 'function' : 'variable'} name does not match the lesson`, + `${lessonLabel} expected ${isFunction ? 'a function' : 'a variable'} named \`${expectedIdentifier}\`, but your code uses \`${similar.actual}\` instead. Python treats uppercase and lowercase letters as different.`, + [ + `Rename \`${similar.actual}\` to \`${expectedIdentifier}\`.`, + `Submit again after the ${isFunction ? 'function' : 'variable'} name matches exactly.` + ] + ); + } + + if (similar?.reason === 'format') { + return createHint( + 'name_mismatch', + 'high', + `The ${isFunction ? 'function' : 'variable'} name is very close, but it still does not match`, + `${lessonLabel} expected ${isFunction ? 'a function' : 'a variable'} named \`${expectedIdentifier}\`, but your code uses \`${similar.actual}\`. Even small differences like missing underscores count as different names in Python.`, + [ + `Rename \`${similar.actual}\` to \`${expectedIdentifier}\`.`, + 'Check the spelling carefully, including underscores.' + ] + ); + } + + return createHint( + isFunction ? 'missing_function' : 'missing_variable', + 'high', + `The lesson could not find the ${isFunction ? 'function' : 'variable'} it expected`, + `${lessonLabel} was looking for ${isFunction ? `a function named \`${expectedIdentifier}\`` : `a variable named \`${expectedIdentifier}\``}, but it could not find one in your code.`, + [ + `Create ${isFunction ? `a function` : `a variable`} named \`${expectedIdentifier}\`.`, + 'Submit again after the required name appears exactly as the lesson expects.' + ] + ); +} + +function buildNameErrorHint( + missingIdentifier: string, + submittedCode: string +): LearnerHint { + const similar = findIdentifierMatch(missingIdentifier, submittedCode); + + if (similar) { + return createHint( + 'name_error', + 'high', + 'Python could not find one of your names', + `Python tried to use \`${missingIdentifier}\`, but that name does not exist. Your code does contain \`${similar.actual}\`, so this is likely a spelling or capitalization mismatch.`, + [ + `Decide whether the name should be \`${missingIdentifier}\` or \`${similar.actual}\`, then make it consistent everywhere.`, + 'Submit again after every use of the name matches exactly.' + ] + ); + } + + return createHint( + 'name_error', + 'medium', + 'Python could not find one of your names', + `Python tried to use \`${missingIdentifier}\`, but that name was never defined before it was used.`, + [ + `Define \`${missingIdentifier}\` before you use it, or fix the spelling if you meant a different name.`, + 'Use the Output tab if you want to see the technical traceback.' + ] + ); +} + +function buildSyntaxHint(message: string): LearnerHint { + return createHint( + 'syntax_error', + 'high', + 'Python could not read your code', + `${message} This usually means Python found a missing quote, colon, parenthesis, or another punctuation problem before the program could run.`, + [ + 'Look closely for missing or extra punctuation near the line mentioned in the Output tab.', + 'After fixing the syntax, submit again.' + ] + ); +} + +function buildIndentationHint(message: string): LearnerHint { + return createHint( + 'indentation_error', + 'high', + 'Python found an indentation problem', + `${message} Lines inside the same block need to line up exactly in Python.`, + [ + 'Check the spaces at the start of each line in the block that failed.', + 'Make sure lines inside loops, functions, and if-statements are indented consistently.' + ] + ); +} + +function buildDataframeMismatchHint(topic: string, operations: string[]): LearnerHint { + const step = describeDataframeStep(topic, operations); + + return createHint( + 'dataframe_mismatch', + 'medium', + 'Your DataFrame result did not match the lesson', + `Your code ran, but the DataFrame it produced while ${step} was different from what the lesson expected.`, + [ + 'Compare the columns, row order, and values in your result.', + `Recheck the step for ${step} and submit again.`, + 'Open Output if you need the technical comparison details.' + ] + ); +} + +function buildVariableValueMismatchHint( + identifier: string, + expected: string, + received: string, + lessonLabel: string +): LearnerHint { + const expectedValue = expected ? `\`${expected}\`` : 'the expected value'; + const receivedValue = received ? `\`${received}\`` : 'a different value'; + + return createHint( + 'output_mismatch', + 'high', + `The variable ${identifier} has the wrong value`, + `${lessonLabel} found the variable \`${identifier}\`, but its value was ${receivedValue} instead of ${expectedValue}.`, + [ + `Set \`${identifier}\` to ${expectedValue}.`, + 'Submit again after that variable matches the lesson exactly.', + 'Open Output if you want to see the technical assertion details.' + ] + ); +} + +function buildOutputMismatchHint(): LearnerHint { + return createHint( + 'output_mismatch', + 'medium', + 'Your program ran, but its result did not match the lesson', + 'Your code finished running, but the final value or printed output was different from what the lesson expected.', + [ + 'Compare the Expected and Actual sections carefully.', + 'Check spelling, spaces, punctuation, and line breaks if the lesson is about printed output.', + 'Open Output if you need the technical test details.' + ] + ); +} + +function extractProgramAssertionIdentifier(diagnosticText: string): string { + const directMatches = [...diagnosticText.matchAll( + /assert\s+program\.([A-Za-z_][A-Za-z0-9_]*)\s*(?:==|is(?:\s+not)?)/gi + )]; + + if (directMatches.length > 0) { + return directMatches[directMatches.length - 1]?.[1] ?? ''; + } + + const whereMatches = [...diagnosticText.matchAll( + /where\s+.+?\.([A-Za-z_][A-Za-z0-9_]*)\b/gi + )]; + + if (whereMatches.length > 0) { + return whereMatches[whereMatches.length - 1]?.[1] ?? ''; + } + + return ''; +} + +function extractAssertionComparisonValues( + diagnosticText: string +): null | { expected: string; received: string } { + const equalityMatches = [...diagnosticText.matchAll( + /AssertionError:[ \t]*assert[ \t]+([^\n]+?)[ \t]*==[ \t]*([^\n]+)/gi + )]; + + if (equalityMatches.length > 0) { + const equalityMatch = equalityMatches[equalityMatches.length - 1]; + return { + received: equalityMatch[1].trim(), + expected: equalityMatch[2].trim(), + }; + } + + const identityMatches = [...diagnosticText.matchAll( + /AssertionError:[ \t]*assert[ \t]+([^\n]+?)[ \t]+is[ \t]+([^\n]+)/gi + )]; + + if (identityMatches.length > 0) { + const identityMatch = identityMatches[identityMatches.length - 1]; + return { + received: identityMatch[1].trim(), + expected: identityMatch[2].trim(), + }; + } + + return null; +} + +function resolveComparisonValues( + detail: FailureDetail, + diagnosticText: string +): null | { expected: string; received: string } { + const structured = { + expected: normalizeInlineValue(detail.expected || ''), + received: normalizeInlineValue(detail.received || ''), + }; + const fallback = extractAssertionComparisonValues(diagnosticText); + const normalizedFallback = fallback + ? { + expected: normalizeInlineValue(fallback.expected), + received: normalizeInlineValue(fallback.received), + } + : null; + const structuredUsable = ( + !isSuspiciousComparisonValue(structured.expected) && + !isSuspiciousComparisonValue(structured.received) + ); + const fallbackUsable = Boolean( + normalizedFallback && + !isSuspiciousComparisonValue(normalizedFallback.expected) && + !isSuspiciousComparisonValue(normalizedFallback.received) + ); + + if (fallbackUsable && !structuredUsable) { + return normalizedFallback; + } + + if (structuredUsable) { + return structured; + } + + if (fallbackUsable) { + return normalizedFallback; + } + + if (structured.expected || structured.received) { + return structured; + } + + return normalizedFallback; +} + +function isLikelyDataframeRuntimeMismatch(diagnosticText: string): boolean { + return ( + /assert_frame_equal|DataFrame|Series/i.test(diagnosticText) || + /has no attribute '(columns|dtypes|shape|index|axes)'/i.test(diagnosticText) || + /(columns|index|shape|dtypes) are different/i.test(diagnosticText) || + /pandas\/core\/indexes\/base\.py|pandas\/core\/computation\/scope\.py/i.test(diagnosticText) || + /\bget_loc\b|UndefinedVariableError|BACKTICK_QUOTED_STRING_/i.test(diagnosticText) || + /None of \[Index\(.+\)\] are in the \[columns\]/i.test(diagnosticText) || + /KeyError:/i.test(diagnosticText) + ); +} + +function buildRuntimeHint(message: string): LearnerHint { + return createHint( + 'runtime_error', + 'medium', + 'Your code ran into an error while it was being checked', + message, + [ + 'Read the Output tab to see where the error happened.', + 'Fix that error first, then submit again.' + ] + ); +} + +function buildUnknownHint(): LearnerHint { + return createHint( + 'unknown', + 'low', + 'The lesson found a problem, but it needs the technical details to explain it', + 'I could not safely turn this failure into a more specific beginner hint yet.', + [ + 'Open the Output tab to see the technical error details.', + 'Focus first on the first error shown there, then submit again.' + ] + ); +} + +function hasAssertionFailure(diagnosticText: string): boolean { + return /AssertionError\b|Assertion failed:|^\s*assert\b/m.test(diagnosticText); +} + +function buildFailureHint(detail: FailureDetail, context: LearnerHintContext): LearnerHint { + const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); + const message = detail.error_message || ''; + const diagnosticText = buildDiagnosticText(detail); + const topicLabel = context.title?.trim() || context.topic?.trim() || 'This lesson'; + const lessonLabel = topicLabel; + const missingAttributeMatch = diagnosticText.match(/module 'program' has no attribute '([A-Za-z_][A-Za-z0-9_]*)'/); + const comparisonValues = resolveComparisonValues(detail, diagnosticText); + + if (missingAttributeMatch) { + const expectedIdentifier = missingAttributeMatch[1]; + const isFunction = contract.expectedFunctions.includes(expectedIdentifier) && + !contract.expectedVariables.includes(expectedIdentifier); + + return buildMissingIdentifierHint(expectedIdentifier, context.submittedCode, lessonLabel, isFunction); + } + + const assertionMissingIdentifierMatch = diagnosticText.match( + /There should be a (variable|function) named exactly ([A-Za-z_][A-Za-z0-9_]*)/i + ); + + if (assertionMissingIdentifierMatch) { + const expectedIdentifier = assertionMissingIdentifierMatch[2]; + const isFunction = assertionMissingIdentifierMatch[1].toLowerCase() === 'function' || + (contract.expectedFunctions.includes(expectedIdentifier) && + !contract.expectedVariables.includes(expectedIdentifier)); + + return buildMissingIdentifierHint(expectedIdentifier, context.submittedCode, lessonLabel, isFunction); + } + + if (/IndentationError:/i.test(diagnosticText)) { + return buildIndentationHint(message || extractFirstMatchingLine(diagnosticText, /IndentationError:[^\n]*/i)); + } + + if (/SyntaxError:/i.test(diagnosticText)) { + return buildSyntaxHint(message || extractFirstMatchingLine(diagnosticText, /SyntaxError:[^\n]*/i)); + } + + if (contract.usesDataframe && isLikelyDataframeRuntimeMismatch(diagnosticText)) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + const nameErrorMatch = diagnosticText.match(/name '([A-Za-z_][A-Za-z0-9_]*)' is not defined/); + if (nameErrorMatch) { + return buildNameErrorHint(nameErrorMatch[1], context.submittedCode); + } + + if (detail.expected || detail.received) { + if (contract.usesDataframe) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + const assertedIdentifier = extractProgramAssertionIdentifier(diagnosticText); + if (assertedIdentifier && contract.expectedVariables.includes(assertedIdentifier)) { + return buildVariableValueMismatchHint( + assertedIdentifier, + comparisonValues?.expected ?? detail.expected, + comparisonValues?.received ?? detail.received, + lessonLabel + ); + } + + return buildOutputMismatchHint(); + } + + if (/AttributeError:|TypeError:|ValueError:|KeyError:|IndexError:/i.test(diagnosticText)) { + return buildRuntimeHint( + message || + extractFirstMatchingLine(diagnosticText, /(AttributeError:|TypeError:|ValueError:|KeyError:|IndexError:)[^\n]*/i) || + 'The lesson reported a runtime error.' + ); + } + + if (hasAssertionFailure(diagnosticText)) { + if (contract.usesDataframe) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + const assertedIdentifier = extractProgramAssertionIdentifier(diagnosticText); + if (assertedIdentifier && contract.expectedVariables.includes(assertedIdentifier)) { + const values = extractAssertionComparisonValues(diagnosticText); + + return buildVariableValueMismatchHint( + assertedIdentifier, + values?.expected ?? '', + values?.received ?? '', + lessonLabel + ); + } + + return buildOutputMismatchHint(); + } + + return buildUnknownHint(); +} + +function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintContext): LearnerHint | null { + const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); + + if (evaluation.failure_details.length > 0) { + return evaluation.failure_details[0].learner_hint ?? null; + } + + if (evaluation.execution_time_exceeded) { + return createHint( + 'timeout', + 'medium', + 'Your code took too long to finish', + 'The lesson stopped your program because it did not finish before the time limit.', + [ + 'Check for loops that never end or code that repeats too much work.', + 'Submit again after your program finishes more quickly.' + ] + ); + } + + if (evaluation.memory_exceeded) { + return createHint( + 'memory_limit', + 'medium', + 'Your code used too much memory', + 'The lesson stopped your program because it tried to store too much data at once.', + [ + 'Look for very large lists, repeated copies of data, or code that keeps growing forever.', + 'Submit again after using less memory.' + ] + ); + } + + if (evaluation.compilation_error) { + return createHint( + 'compile_error', + 'medium', + 'Your code could not be compiled', + evaluation.compilation_error, + [ + 'Fix the first compiler error shown in Output.', + 'Submit again after the code compiles successfully.' + ] + ); + } + + if (evaluation.runtime_error) { + if (/IndentationError:/i.test(evaluation.runtime_error)) { + return buildIndentationHint(evaluation.runtime_error); + } + + if (/SyntaxError:/i.test(evaluation.runtime_error)) { + return buildSyntaxHint(evaluation.runtime_error); + } + + if (contract.usesDataframe && isLikelyDataframeRuntimeMismatch(evaluation.runtime_error)) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + return buildRuntimeHint(evaluation.runtime_error); + } + + if (evaluation.state === 'passed') { + return null; + } + + return buildUnknownHint(); +} + +function addLearnerHintsToEvaluation( + evaluation: EvaluationResponse, + context: LearnerHintContext +): EvaluationResponse { + const failure_details = evaluation.failure_details.map((detail) => ({ + ...detail, + learner_hint: buildFailureHint(detail, context), + })); + + const hintedEvaluation = { + ...evaluation, + failure_details, + }; + + return { + ...hintedEvaluation, + learner_hint: buildTopLevelHint(hintedEvaluation, context), + }; +} + +export type { + LearnerHintContext, +}; + +export { + addLearnerHintsToEvaluation, +}; diff --git a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx new file mode 100644 index 0000000..9d2fa69 --- /dev/null +++ b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx @@ -0,0 +1,90 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, describe, expect, it } from 'vitest'; +import type { EvaluationResponse } from '../../interfaces/evaluation'; +import CodeSubmission from './CodeSubmission'; + +afterEach(() => { + cleanup(); +}); + +describe('CodeSubmission', () => { + it('shows the learner hint in Outcome and keeps technical details in Output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'numberOfHats'", + rawout: "E AttributeError: module 'program' has no attribute 'numberOfHats'", + learner_hint: { + kind: 'name_mismatch', + confidence: 'high', + title: 'The variable name does not match the lesson', + summary: 'This lesson expected numberOfHats, but your code used NumberOfHats.', + next_steps: ['Rename NumberOfHats to numberOfHats.'], + }, + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + learner_hint: { + kind: 'name_mismatch', + confidence: 'high', + title: 'The variable name does not match the lesson', + summary: 'This lesson expected numberOfHats, but your code used NumberOfHats.', + next_steps: ['Rename NumberOfHats to numberOfHats.'], + }, + }; + + render(); + + expect(screen.getByText('The variable name does not match the lesson')).toBeTruthy(); + expect(screen.queryByText(/AttributeError/)).toBeNull(); + + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/AttributeError/)).toBeTruthy(); + }); + + it('shows a top-level learner hint for technical errors without failure details', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [], + compilation_error: '', + runtime_error: 'SyntaxError: invalid syntax', + execution_time_exceeded: false, + memory_exceeded: false, + learner_hint: { + kind: 'syntax_error', + confidence: 'high', + title: 'Python could not read your code', + summary: 'SyntaxError: invalid syntax', + next_steps: ['Fix the syntax problem and submit again.'], + }, + }; + + render(); + + expect(screen.getByText('Python could not read your code')).toBeTruthy(); + + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText('SyntaxError: invalid syntax')).toBeTruthy(); + }); +}); diff --git a/codewit/client/src/components/codeblock/CodeSubmission.tsx b/codewit/client/src/components/codeblock/CodeSubmission.tsx index 0fbfdfe..b964526 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.tsx @@ -1,11 +1,42 @@ import { BiSolidRightArrow, BiSolidLeftArrow } from 'react-icons/bi'; import { useState } from 'react'; import type { EvaluationResponse } from '../../interfaces/evaluation'; +import type { LearnerHint } from '@codewit/interfaces'; type EvalProps = { evaluation: EvaluationResponse | null; }; +const fallbackHint: LearnerHint = { + kind: 'unknown', + confidence: 'low', + title: 'The lesson found a problem', + summary: 'Open the Output tab to see the technical details, then fix the first error shown there.', + next_steps: [ + 'Read the first technical error in Output.', + 'Fix that error and submit again.' + ], +}; + +const HintCard = ({ hint }: { hint: LearnerHint }): JSX.Element => { + return ( +
+

{hint.title}

+

{hint.summary}

+ {hint.next_steps.length > 0 && ( +
+
Try this next:
+
    + {hint.next_steps.map((step) => ( +
  • {step}
    • + ))} +
+
+ )} +
+ ); +}; + const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const [activeTab, setActiveTab] = useState<'outcome' | 'output'>('outcome'); const [issueIdx, setIssueIdx] = useState(0); @@ -30,20 +61,15 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { memory_exceeded = false, } = evaluation; const error = 'error' in evaluation ? evaluation.error : ''; - const rawout = failure_details[issueIdx]?.rawout || ''; - - let errorMessage = null; - if (compilation_error) errorMessage = compilation_error; - else if (runtime_error) errorMessage = runtime_error; - else if (execution_time_exceeded) errorMessage = 'Execution time exceeded'; - else if (memory_exceeded) errorMessage = 'Memory limit exceeded'; - else if (state === 'error') errorMessage = 'Evaluation failed ' + error; + const activeIssue = failure_details[issueIdx] || null; + const topLevelHint = 'learner_hint' in evaluation ? (evaluation.learner_hint ?? null) : null; + const activeHint = activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint); + const technicalOutput = activeIssue?.rawout || compilation_error || runtime_error || error || ''; const hasFailures = failure_details.length > 0; - const hasOutput = rawout.trim().length > 0; - const allPassed = !errorMessage && !hasFailures && state === 'passed'; - const activeIssue = failure_details[issueIdx] || null; - const showOutcomeTab = hasFailures || !errorMessage; + const hasOutput = technicalOutput.trim().length > 0; + const allPassed = !hasFailures && !compilation_error && !runtime_error && !execution_time_exceeded && !memory_exceeded && !error && state === 'passed'; + const showOutcomeTab = true; return (
@@ -75,7 +101,6 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
)} {allPassed && All tests passed!} - {errorMessage && 
{errorMessage}
}
@@ -108,10 +133,12 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
{activeTab === 'outcome' && showOutcomeTab ? ( <> + {!allPassed && activeHint && ( + + )} {hasFailures && activeIssue && (
- {activeIssue.test_case}
- {activeIssue.error_message} + {activeIssue.test_case} {activeIssue.expected && (
Expected: @@ -124,18 +151,19 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { 
{activeIssue.received}
)} +

Open the Output tab to see the technical details for this issue.

)} - {!errorMessage && !hasFailures && ( + {!allPassed && !hasFailures && !activeHint && (
- No test cases to show. + Open the Output tab to see the technical details.
)} ) : ( activeTab === 'output' && hasOutput && (
- 
{rawout}
+ 
{technicalOutput}
) )} diff --git a/codewit/client/src/interfaces/evaluation.ts b/codewit/client/src/interfaces/evaluation.ts index e6c422c..0a82bcc 100644 --- a/codewit/client/src/interfaces/evaluation.ts +++ b/codewit/client/src/interfaces/evaluation.ts @@ -1,4 +1,4 @@ -import type { AttemptDTO, FailureDetail, TestResult } from '@codewit/interfaces'; +import type { AttemptDTO, FailureDetail, LearnerHint, TestResult } from '@codewit/interfaces'; export interface EvaluationErrorResponse { state: 'error'; @@ -14,6 +14,7 @@ export interface EvaluationErrorResponse { execution_time_exceeded: false; memory_exceeded: false; error: string; + learner_hint?: LearnerHint | null; } export type EvaluationResponse = TestResult | EvaluationErrorResponse; diff --git a/codewit/lib/shared/interfaces/src/lib/output.ts b/codewit/lib/shared/interfaces/src/lib/output.ts index 2476c09..acb0baf 100644 --- a/codewit/lib/shared/interfaces/src/lib/output.ts +++ b/codewit/lib/shared/interfaces/src/lib/output.ts @@ -6,6 +6,31 @@ type EvaluationState = | 'execution_error' | 'execution_blocked'; +type LearnerHintConfidence = 'high' | 'medium' | 'low'; + +type LearnerHintKind = + | 'missing_variable' + | 'missing_function' + | 'name_error' + | 'name_mismatch' + | 'output_mismatch' + | 'syntax_error' + | 'indentation_error' + | 'dataframe_mismatch' + | 'compile_error' + | 'runtime_error' + | 'timeout' + | 'memory_limit' + | 'unknown'; + +interface LearnerHint { + kind: LearnerHintKind; + confidence: LearnerHintConfidence; + title: string; + summary: string; + next_steps: string[]; +} + interface FailureDetail { test_case: string | number; expected: string; @@ -13,6 +38,7 @@ interface FailureDetail { error_message: string; rawout: string; stderr?: string; + learner_hint?: LearnerHint; } interface TestResult { @@ -28,10 +54,14 @@ interface TestResult { runtime_error: string; execution_time_exceeded: boolean; memory_exceeded: boolean; + learner_hint?: LearnerHint | null; } export type { EvaluationState, + LearnerHint, + LearnerHintConfidence, + LearnerHintKind, FailureDetail, TestResult, };