Add beginner-friendly evaluation feedback for Python lessons

codewit/api/src/controllers/attempt.ts

@@ -4,6 +4,7 @@ import { UserExerciseCompletion } from '../models/userExerciseCompletion';
 import { UserModuleCompletion } from '../models/userModuleCompletion';
 import { AttemptWithEval } from '../typings/response.types';
 import { EvaluationPayload, EvaluationResponse, executeCodeEvaluation } from '../utils/codeEvalService';
+import { addLearnerHintsToEvaluation } from '../utils/learnerHints';
 import { Language as LanguageEnum } from '@codewit/language';
 
 function getEvaluationError(response: EvaluationResponse): string {
@@ -83,9 +84,14 @@ async function createAttempt(
 
     let evalResponse: EvaluationResponse | null = null;
     try {
-      const response = await executeCodeEvaluation(evaluationPayload, cookies);
+      const rawResponse = await executeCodeEvaluation(evaluationPayload, cookies);
+      const response = addLearnerHintsToEvaluation(rawResponse, {
+        referenceTest: exercise.referenceTest,
+        submittedCode: code,
+        topic: exercise.topic,
+        title: exercise.title,
+      });
       evalResponse = response;
-      console.log('Code evaluation response:', response);
       const { tests_run, passed } = response;
       const evalError = getEvaluationError(response);
 
@@ -95,7 +101,6 @@ async function createAttempt(
       if (tests_run > 0) {
         const completionPercentage = Math.round((passed / tests_run) * 100);
         attempt.completionPercentage = completionPercentage;
-        console.log(`Completion Percentage: ${completionPercentage}%`);
 
         // Update UserExerciseCompletion
         const completion = passed / tests_run;
@@ -184,7 +189,8 @@ async function createAttempt(
         console.warn('Code evaluation returned a passed state without runnable tests:', response);
       }
     } catch (err) {
-      console.error('Code evaluation failed:', err.message);
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      console.error('Code evaluation failed:', errorMessage);
       throw new Error('Code evaluation failed');
     }
 

codewit/api/src/models/attempt.ts

@@ -12,6 +12,7 @@ import {
   Model,
   InferAttributes,
   InferCreationAttributes,
+  CreationOptional,
   DataTypes,
   Sequelize,
   NonAttribute,
@@ -24,14 +25,14 @@ class Attempt extends Model<
   InferAttributes<Attempt>,
   InferCreationAttributes<Attempt>
 > {
-  declare uid: number;
-  declare timestamp: Date;
+  declare uid: CreationOptional<number>;
+  declare timestamp: CreationOptional<Date>;
   declare exercise: NonAttribute<Exercise>;
   declare user: NonAttribute<User>;
-  declare submissionNumber: number;
+  declare submissionNumber: CreationOptional<number>;
   declare code: string;
-  declare completionPercentage: number;
-  declare error: string;
+  declare completionPercentage: CreationOptional<number>;
+  declare error: CreationOptional<null | string>;
   declare exerciseUid: number;
   declare userUid: number;
 

codewit/api/src/utils/exerciseContract.spec.ts

@@ -0,0 +1,300 @@
+import { extractExerciseContract } from './exerciseContract';
+
+describe('extractExerciseContract', () => {
+  it('detects variable expectations from lesson tests', () => {
+    const contract = extractExerciseContract(
+      `
+import program
+
+def test_hat_variables():
+    assert program.numberOfHats == 9
+      `.trim(),
+      'variable',
+      'Collecting Hats'
+    );
+
+    expect(contract.lessonFamily).toBe('variable');
+    expect(contract.expectedVariables).toContain('numberOfHats');
+    expect(contract.expectedFunctions).toHaveLength(0);
+  });
+
+  it('detects variable expectations from hasattr checks', () => {
+    const contract = extractExerciseContract(
+      `
+import sys
+
+def test_hat_variables():
+    sys.modules.pop("program", None)
+    import program
+    assert hasattr(program, "HatName")
+      `.trim(),
+      'variable',
+      'Collecting Hats'
+    );
+
+    expect(contract.lessonFamily).toBe('variable');
+    expect(contract.expectedVariables).toContain('HatName');
+    expect(contract.expectedFunctions).toHaveLength(0);
+  });
+
+  it('detects aliased program functions from live lesson tests', () => {
+    const contract = extractExerciseContract(
+      `
+import sys
+
+def test_choose_clothes_function_variants():
+    sys.modules.pop("program", None)
+    import program
+    f = program.chooes_clothes
+    assert f("weds") == {"shirt": "pink"}
+      `.trim(),
+      'function',
+      'Wardrobe rules'
+    );
+
+    expect(contract.lessonFamily).toBe('function');
+    expect(contract.expectedFunctions).toContain('chooes_clothes');
+    expect(contract.expectedVariables).not.toContain('chooes_clothes');
+  });
+
+  it('detects dataframe operations and expected functions', () => {
+    const contract = extractExerciseContract(
+      `
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import program
+
+def test_load_dataframe():
+    expected = pd.read_csv('world_cup.csv')
+    result = program.loadWorldCupData()
+    assert_frame_equal(result, expected)
+      `.trim(),
+      'load dataframe',
+      'Load World cup data'
+    );
+
+    expect(contract.lessonFamily).toBe('dataframe');
+    expect(contract.usesDataframe).toBe(true);
+    expect(contract.expectedFunctions).toContain('loadWorldCupData');
+    expect(contract.dataframeOperations).toContain('load_dataframe');
+  });
+
+  it('detects console io lessons from production-style tests', () => {
+    const contract = extractExerciseContract(
+      `
+from pytest import MonkeyPatch
+import builtins
+import sys
+
+def run_program_with(monkeypatch, inputs):
+    monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs))
+    sys.modules.pop("program", None)
+    import program
+
+def test_song_request(monkeypatch, capsys):
+    user_input = iter(["Hello"])
+    run_program_with(monkeypatch, user_input)
+    captured = capsys.readouterr().out
+    expected_output = "What song would you like to add\\n"
+    assert captured == expected_output
+      `.trim(),
+      'console io',
+      'Song Request'
+    );
+
+    expect(contract.lessonFamily).toBe('console_io');
+    expect(contract.usesInput).toBe(true);
+    expect(contract.usesConsoleOutput).toBe(true);
+  });
+
+  [
+    {
+      topic: 'describe dataframe',
+      title: 'Statistics of hair product data',
+      referenceTest: `
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import program
+
+def test_describe_dataframe():
+    source = pd.read_csv('./datasets/hair.csv')
+    expected = source.describe()
+    result = program.describeHairProducts()
+    assert_frame_equal(result, expected)
+      `.trim(),
+      operation: 'describe_dataframe',
+      expectedFunction: 'describeHairProducts',
+    },
+    {
+      topic: 'query dataframe',
+      title: 'Search hair product data',
+      referenceTest: `
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import program
+
+def test_query_dataframe():
+    source = pd.read_csv('./datasets/hair.csv')
+    expected = source.query("Brand == 'Eco Style'")
+    result = program.searchHairProducts()
+    assert_frame_equal(result, expected)
+      `.trim(),
+      operation: 'query_dataframe',
+      expectedFunction: 'searchHairProducts',
+    },
+    {
+      topic: 'melt dataframe',
+      title: 'Unpivot hair data',
+      referenceTest: `
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import program
+
+def test_melt_dataframe():
+    source = pd.read_csv('./datasets/hair.csv')
+    expected = pd.melt(source, id_vars=['Brand'])
+    result = program.unpivotHairData()
+    assert_frame_equal(result, expected)
+      `.trim(),
+      operation: 'melt_dataframe',
+      expectedFunction: 'unpivotHairData',
+    },
+    {
+      topic: 'pivot dataframe',
+      title: 'Pivot hair product data',
+      referenceTest: `
+import pandas as pd
+from pandas.testing import assert_frame_equal
+import program
+
+def test_pivot_dataframe():
+    source = pd.read_csv('./datasets/hair.csv')
+    expected = source.pivot_table(index='Brand', values='Price', aggfunc='mean')
+    result = program.pivotHairProducts()
+    assert_frame_equal(result, expected)
+      `.trim(),
+      operation: 'pivot_dataframe',
+      expectedFunction: 'pivotHairProducts',
+    },
+  ].forEach(({ topic, title, referenceTest, operation, expectedFunction }) => {
+    it(`detects dataframe operation ${operation} for ${title}`, () => {
+      const contract = extractExerciseContract(referenceTest, topic, title);
+
+      expect(contract.lessonFamily).toBe('dataframe');
+      expect(contract.usesDataframe).toBe(true);
+      expect(contract.dataframeOperations).toContain(operation);
+      expect(contract.expectedFunctions).toContain(expectedFunction);
+    });
+  });
+
+  describe('regression cases', () => {
+    it('detects lowercase snake_case variable expectations from player stat lessons', () => {
+      const contract = extractExerciseContract(
+        `
+import sys
+
+def test_player_name():
+    sys.modules.pop("program", None)
+    import program
+    assert program.player_name == "Coolminivan"
+
+def test_is_player_on():
+    assert program.is_player_on is True
+
+def test_level_player():
+    assert program.level_player == 64
+    assert isinstance(program.level_player, int)
+        `.trim(),
+        'variable',
+        'Player Stats'
+      );
+
+      expect(contract.lessonFamily).toBe('variable');
+      expect(contract.expectedVariables).toEqual(
+        expect.arrayContaining(['player_name', 'is_player_on', 'level_player'])
+      );
+    });
+
+    it('detects printed-output lessons that do not use input', () => {
+      const contract = extractExerciseContract(
+        `
+import sys
+
+def test_outputs_no_input(capsys):
+    sys.modules.pop("program", None)
+    import program
+    out = capsys.readouterr().out
+    expected = (
+        "480\\n"
+        "285\\n"
+        "405\\n"
+        "2.7857142857142856\\n"
+        "9\\n"
+        "9\\n"
+    )
+    assert out == expected
+        `.trim(),
+        'math operation',
+        'Road Trip Math'
+      );
+
+      expect(contract.lessonFamily).toBe('general');
+      expect(contract.usesConsoleOutput).toBe(true);
+      expect(contract.usesInput).toBe(false);
+    });
+
+    it('detects printed dataframe lessons from topic metadata even without assert_frame_equal', () => {
+      const contract = extractExerciseContract(
+        `
+import importlib
+import sys
+
+def test_describe_printed(capsys):
+    sys.modules.pop("program", None)
+    import program
+    importlib.reload(program)
+    out = capsys.readouterr().out.strip()
+    expected_parts = ["count", "mean", "std", "min", "max"]
+    for part in expected_parts:
+        assert part in out, f"Expected '{part}' in output, got:\\n{out}"
+        `.trim(),
+        'describe dataframe',
+        'Statistics of hair product data'
+      );
+
+      expect(contract.lessonFamily).toBe('dataframe');
+      expect(contract.usesDataframe).toBe(true);
+      expect(contract.usesConsoleOutput).toBe(true);
+      expect(contract.dataframeOperations).toContain('describe_dataframe');
+    });
+
+    it('does not treat program.py file references as an expected learner variable', () => {
+      const contract = extractExerciseContract(
+        `
+import importlib
+import sys
+import os
+import pytest
+
+def test_program_output(capsys):
+    if not os.path.exists("./datasets/worldcup.csv"):
+        pytest.skip("Dataset not found; skipping test")
+
+    sys.modules.pop("program", None)
+    import program
+    importlib.reload(program)
+
+    out = capsys.readouterr().out.strip()
+    assert out, "No output printed from program.py"
+    assert "RUNNER UP" in out
+        `.trim(),
+        'load dataframe',
+        'World Cup Load Dataframe'
+      );
+
+      expect(contract.lessonFamily).toBe('dataframe');
+      expect(contract.usesDataframe).toBe(true);
+      expect(contract.expectedVariables).not.toContain('py');
+    });
+  });
+});