eval.py

"""Eval harness: does Cortex keep the SIGNAL and forget the NOISE, while staying
within a context budget? This targets the MemoryAgent rubric's "recall within
context limits" + "forgetting stale info" criteria. No API key (mock embeddings).

Run:  python3 eval.py
"""
import random
import time

from cortex.memory_engine import MemoryEngine

random.seed(7)
DAY = 86400

IMPORTANT = [
    ("The user is allergic to penicillin", "fact", 0.95,
     "is the user allergic to penicillin"),
    ("The user is a vegetarian", "preference", 0.9,
     "is the user a vegetarian"),
    ("The user's daughter is named Aisha", "fact", 0.9,
     "what is the user's daughter named Aisha"),
    ("The user works as a civil engineer", "fact", 0.85,
     "does the user work as a civil engineer"),
    ("The user prefers metric units", "preference", 0.8,
     "does the user prefer metric units"),
]
NOISE = ["The user said the weather was {} today",
         "The user mentioned traffic was {} this morning",
         "The user typed a {} hello",
         "The user joked about something {} once"]
WORDS = ["sunny", "rainy", "heavy", "light", "cheerful", "random", "mild", "grey", "busy", "quiet"]
NOISE_N = 40

eng = MemoryEngine()
for content, kind, sal, _ in IMPORTANT:
    eng.add(content, kind, sal)
for i in range(NOISE_N):
    eng.add(random.choice(NOISE).format(random.choice(WORDS)) + f" #{i}", "event",
            round(random.uniform(0.1, 0.3), 2), dedupe=False)

total = len(eng.memories)
print(f"Seeded {len(IMPORTANT)} important facts + {NOISE_N} noisy one-offs = {total} memories.")

# simulate ~4 weeks passing
shift = 28 * DAY
for m in eng.memories:
    m.created_at -= shift
    m.last_accessed -= shift

held_before = len(eng.memories)
eng.forget()
held_after = len(eng.memories)

important_held = sum(1 for m in eng.memories if any(m.content == c for c, *_ in IMPORTANT))
noise_held = held_after - important_held

# recall test: can we still retrieve each important fact?
recalled = 0
for content, _, _, q in IMPORTANT:
    hits = eng.retrieve(q, k=3)
    if any(h.content == content for h in hits):
        recalled += 1

print("\n--- after 4 weeks + forgetting pass ---")
print(f"Memories held:        {held_before} -> {held_after}  (context footprint shrank {round(100*(1-held_after/held_before))}%)")
print(f"Important retained:   {important_held}/{len(IMPORTANT)}")
print(f"Noise forgotten:      {NOISE_N - noise_held}/{NOISE_N}  ({round(100*(NOISE_N-noise_held)/NOISE_N)}%)")
print(f"Recall of importants: {recalled}/{len(IMPORTANT)}  (recall@3)")
print("\nPASS" if recalled == len(IMPORTANT) and noise_held <= 2 else "\nCHECK THRESHOLDS")