From 1586360e31cb51bb0e41530efedf25c72836e863 Mon Sep 17 00:00:00 2001 From: Siddhant Singh Date: Wed, 26 Aug 2026 13:28:31 +0000 Subject: [PATCH 1/3] fix(recall): recover community participation sets from joined activities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enumerate joined/attended/organizing/mentorship objects for "in what ways … community" questions so leftover courage slogans cannot crowd out the typed participation list. Co-authored-by: aria --- internal/memory/hop_executor.go | 137 +++++++++++++++++++++++++++++++- internal/memory/recall.go | 51 +++++++++++- internal/memory/recall_test.go | 137 ++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+), 4 deletions(-) diff --git a/internal/memory/hop_executor.go b/internal/memory/hop_executor.go index c0c3849..2eadc84 100644 --- a/internal/memory/hop_executor.go +++ b/internal/memory/hop_executor.go @@ -790,7 +790,8 @@ func (s *Service) recoverSlotAlignedHops(ctx context.Context, tenantID, subjectI needNames := looksNameListQuery(query) needFood := looksFoodSetQuery(query) needBeneficiaries := looksBeneficiarySetQuery(query) - if !needLoc && !needUnwind && !needPlay && !needTrick && !needItems && !needBesides && !needNames && !needFood && !needBeneficiaries { + needParticipation := looksParticipationSetQuery(query) + if !needLoc && !needUnwind && !needPlay && !needTrick && !needItems && !needBesides && !needNames && !needFood && !needBeneficiaries && !needParticipation { return } listed, err := s.store.ListMemories(ctx, tenantID, subjectID, false) @@ -854,6 +855,19 @@ func (s *Service) recoverSlotAlignedHops(ctx context.Context, tenantID, subjectI } } } + if needParticipation { + slots := recoverParticipationSlots(person, listed) + if len(slots) >= 2 { + idx := hopIndexForPredicateEntity(hops, PredicateActivity, person) + replaceHopSlotsOn(hops, idx, slots) + if idx >= 0 && idx < len(hops) && len(hops[idx].Values) >= 2 { + hops[idx].Predicate = PredicateActivity + if person != "" { + hops[idx].Entity = person + } + } + } + } } func prependHopSlots(hops []HopResult, pred string, slots []recoveredSlot) { @@ -1547,6 +1561,127 @@ func looksBeneficiarySetQuery(query string) bool { return strings.HasPrefix(q, "who") || strings.HasPrefix(q, "which") || strings.HasPrefix(q, "what") } +func looksParticipationSetQuery(query string) bool { + q := strings.ToLower(strings.TrimSpace(query)) + if q == "" || !strings.Contains(q, "community") { + return false + } + return strings.Contains(q, "ways") || strings.HasPrefix(q, "in what") +} + +func looksThinParticipation(v string) bool { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "" { + return true + } + switch v { + case "community", "rights", "difference", "painting", "sidewalk", "month", + "city", "neighborhood", "students", "folks", "adoption", "freedom", + "pride", "work", "people", "group", "events", "campaigns", "meetings": + return true + } + if strings.HasPrefix(v, "community ") && !strings.Contains(v, "garden") { + return true + } + return false +} + +func recoverParticipationSlots(person string, listed []MemoryRecord) []recoveredSlot { + if person == "" { + return nil + } + var out []recoveredSlot + seen := map[string]struct{}{} + add := func(sl recoveredSlot) { + val := strings.TrimSpace(sl.value) + val = strings.Trim(val, ".,;: ") + if val == "" || anaphoricSlotValue(val) || looksCodedSlotValue(val) || utf8Len(val) > 48 || looksThinParticipation(val) { + return + } + key := strings.ToLower(val) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + out = append(out, sl) + } + for _, rec := range listed { + content := strings.TrimSpace(rec.Content) + if content == "" || strings.HasPrefix(content, "[") { + continue + } + if !strings.EqualFold(entitySubjectOf(rec), person) && !queryHasToken(content, person) && + !strings.HasPrefix(strings.ToLower(content), strings.ToLower(person)+":") { + continue + } + for _, v := range participationObjectsFromContent(content) { + add(recoveredSlot{value: v, content: content, memID: rec.MemoryID}) + } + } + if len(out) > 8 { + out = out[:8] + } + return out +} + +func participationObjectsFromContent(content string) []string { + lower := strings.ToLower(content) + var out []string + seen := map[string]struct{}{} + add := func(v string) { + v = strings.TrimSpace(v) + v = strings.Trim(v, ".,;: ") + if j := strings.IndexAny(v, ".!?"); j >= 0 { + v = strings.TrimSpace(v[:j]) + } + for _, tail := range []string{" which ", " during ", " on ", " last ", " after ", " and we ", " for ", " since ", " scheduled "} { + if k := strings.Index(strings.ToLower(v), tail); k >= 3 { + v = strings.TrimSpace(v[:k]) + } + } + if low := strings.ToLower(v); strings.HasSuffix(low, " scheduled") { + v = strings.TrimSpace(v[:len(v)-len(" scheduled")]) + } + v = strings.TrimPrefix(v, "a ") + v = strings.TrimPrefix(v, "an ") + v = strings.TrimPrefix(v, "the ") + v = strings.TrimPrefix(v, "new ") + if v == "" || utf8Len(v) < 4 || utf8Len(v) > 48 || looksThinParticipation(v) { + return + } + key := strings.ToLower(v) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + out = append(out, v) + } + for _, cue := range []string{ + " joined a ", " joined an ", " joined the ", " joined ", + " attended a ", " attended an ", " attended the ", " attended ", + " organizing an ", " organizing a ", " host an ", " host a ", " hosting an ", " hosting a ", + "mentorship program", + } { + start := 0 + for { + i := strings.Index(lower[start:], cue) + if i < 0 { + break + } + i += start + if cue == "mentorship program" { + add("mentorship program") + start = i + len(cue) + continue + } + rest := strings.TrimSpace(content[i+len(cue):]) + add(rest) + start = i + len(cue) + } + } + return out +} + func looksThinBeneficiary(v string) bool { v = strings.ToLower(strings.TrimSpace(v)) if v == "" { diff --git a/internal/memory/recall.go b/internal/memory/recall.go index 0edd59e..9483541 100644 --- a/internal/memory/recall.go +++ b/internal/memory/recall.go @@ -1190,7 +1190,7 @@ func wantsTypedSetScan(query string) bool { // current-state for possession/skill/place sets. Activity/community/snack // leftover questions enumerate at search top-k without this widening. func wantsHistoricalAtomScan(query string) bool { - if looksBeneficiarySetQuery(query) { + if looksBeneficiarySetQuery(query) || looksParticipationSetQuery(query) { return true } q := strings.ToLower(strings.TrimSpace(query)) @@ -1311,7 +1311,7 @@ func (s *Service) enumerateFromSearch(ctx context.Context, req RecallRequest, re // Meal/suggestion sets are one person's preference slots, not the // intersection of giver and recipient (or eater and clause entity). // Beneficiary org lists stay on the named person's affiliations. - if looksFoodSetQuery(req.Query) || looksBeneficiarySetQuery(req.Query) { + if looksFoodSetQuery(req.Query) || looksBeneficiarySetQuery(req.Query) || looksParticipationSetQuery(req.Query) { join = false } if join { @@ -1376,13 +1376,21 @@ func (s *Service) enumerateFromSearch(ctx context.Context, req RecallRequest, re continue } } + if looksParticipationSetQuery(req.Query) { + if !strings.EqualFold(h.Predicate, PredicateActivity) { + continue + } + } if !hopUsefulForEnumerate(h.Predicate, pred, counting) { continue } slotPred := firstNonEmpty(h.Predicate, pred) if len(h.Values) > 0 { for i, v := range h.Values { - if hopValueIsAttendedEvent(v) || hopValueHasForeignPossessive(v, h.Entity, hops) { + if !looksParticipationSetQuery(req.Query) && hopValueIsAttendedEvent(v) { + continue + } + if hopValueHasForeignPossessive(v, h.Entity, hops) { continue } id := hopMemoryIDForExtractedValue(h, v) @@ -2065,6 +2073,15 @@ func (s *Service) refineEnumeratedItems(ctx context.Context, req RecallRequest, return capEnumerateItems(kept) } } + if looksParticipationSetQuery(req.Query) { + person := "" + if ents := hopQueryEntities(req.Query); len(ents) > 0 { + person = ents[0] + } + if kept := itemsOnPredicateHop(items, hops, PredicateActivity, person); len(kept) >= 2 { + return capEnumerateItems(kept) + } + } if !looksCountQuery(req.Query) { if toks := forClauseTokens(req.Query); len(toks) > 0 && predicateFromListQuery(tokenize(req.Query)) != PredicateFamilyMember { items = dropForClauseClassReferents(items, toks) @@ -10471,6 +10488,31 @@ func leftoverCoveringKeepBeneficiarySet(query string, hops []HopResult, answer s return hits >= 2 && utf8Len(answer) <= 240 } +func leftoverCoveringKeepParticipationSet(query string, hops []HopResult, answer string) bool { + if !looksParticipationSetQuery(query) { + return false + } + answer = strings.TrimSpace(answer) + if answer == "" || strings.EqualFold(answer, "not in memory") { + return false + } + if leftoverThinMissAnswer(query, hops, answer) || leftoverQueryEchoAnswer(query, answer) { + return false + } + vals := hopPredicateValues(hops, PredicateActivity, "") + if len(vals) < 2 { + return false + } + al := strings.ToLower(answer) + hits := 0 + for _, v := range vals { + if utf8.RuneCountInString(v) >= 4 && strings.Contains(al, strings.ToLower(v)) { + hits++ + } + } + return hits >= 2 && utf8Len(answer) <= 400 +} + func leftoverCoveringKeepTypedAnswer(query string, hops []HopResult, answer string) bool { if looksWhereQuery(query) || leftoverCoveringShouldJoin(query) { return false @@ -10478,6 +10520,9 @@ func leftoverCoveringKeepTypedAnswer(query string, hops []HopResult, answer stri if leftoverCoveringKeepBeneficiarySet(query, hops, answer) { return true } + if leftoverCoveringKeepParticipationSet(query, hops, answer) { + return true + } if !hopsKeepTypedJoin(hops) { return false } diff --git a/internal/memory/recall_test.go b/internal/memory/recall_test.go index 4d3a863..d498d47 100644 --- a/internal/memory/recall_test.go +++ b/internal/memory/recall_test.go @@ -2483,6 +2483,132 @@ func TestLooksBeneficiarySetQueryAndObjects(t *testing.T) { } } +func TestRecallParticipationSetFromJoinedActivities(t *testing.T) { + t.Setenv("BRAINY_RECALL_LLM", "") + store := newMemoryStoreStub() + svc := NewService(store) + now := svc.now() + store.records["c-act"] = MemoryRecord{ + MemoryID: "mem_cact", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline joined an LGBTQ activist group on 2023-07-18.", + DedupeKey: "cact", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateAffiliation, "value_norm": "lgbtq activist group", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateAffiliation, "value_norm": "lgbtq activist group", "subject": "Caroline"}, + } + store.records["c-pride"] = MemoryRecord{ + MemoryID: "mem_cpride", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline attended an LGBTQ+ pride parade on 2023-06-30.", + DedupeKey: "cpride", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateEvent, "value_norm": "pride parade", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateEvent, "value_norm": "pride parade", "subject": "Caroline"}, + } + store.records["c-art"] = MemoryRecord{ + MemoryID: "mem_cart", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline is organizing an LGBTQ art show scheduled for September 2023.", + DedupeKey: "cart", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "organizing lgbtq art show", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "organizing lgbtq art show", "subject": "Caroline"}, + } + store.records["c-ment"] = MemoryRecord{ + MemoryID: "mem_cment", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline joined a mentorship program for LGBTQ youth on the weekend of 15–16 July 2023.", + DedupeKey: "cment", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "joined lgbtq youth mentorship program", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "joined lgbtq youth mentorship program", "subject": "Caroline"}, + } + store.records["c-hike"] = MemoryRecord{ + MemoryID: "mem_chike", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline participates in hiking", + DedupeKey: "chike", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "hiking", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "hiking", "subject": "Caroline"}, + } + store.records["c-job"] = MemoryRecord{ + MemoryID: "mem_cjob", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline works as a nurse", + DedupeKey: "cjob", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateOccupation, "value_norm": "nurse", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateOccupation, "value_norm": "nurse", "subject": "Caroline"}, + } + store.records["c-adv"] = MemoryRecord{ + MemoryID: "mem_cadv", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline contacted her mentor for adoption advice.", + DedupeKey: "cadv", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "contacted mentor for adoption advice", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "contacted mentor for adoption advice", "subject": "Caroline"}, + } + store.atoms = append(store.atoms, + stubAtom{pred: PredicateAffiliation, val: "lgbtq activist group", memID: "mem_cact"}, + stubAtom{pred: PredicateEvent, val: "pride parade", memID: "mem_cpride"}, + stubAtom{pred: PredicateActivity, val: "organizing lgbtq art show", memID: "mem_cart"}, + stubAtom{pred: PredicateActivity, val: "joined lgbtq youth mentorship program", memID: "mem_cment"}, + stubAtom{pred: PredicateActivity, val: "hiking", memID: "mem_chike"}, + stubAtom{pred: PredicateOccupation, val: "nurse", memID: "mem_cjob"}, + stubAtom{pred: PredicateActivity, val: "contacted mentor for adoption advice", memID: "mem_cadv"}, + ) + out, err := svc.Recall(context.Background(), RecallRequest{ + TenantID: "t-part", SubjectID: "u1", + Query: "In what ways is Caroline participating in the LGBTQ community?", Mode: "answer", TopK: 20, + }) + if err != nil { + t.Fatal(err) + } + got := strings.ToLower(out.Answer) + if !strings.Contains(got, "activist") || !strings.Contains(got, "pride") || !strings.Contains(got, "art") || !strings.Contains(got, "mentor") { + t.Fatalf("expected joined/attended/organizing/mentorship set, answer=%q hops=%v", out.Answer, out.Explain["hop_results"]) + } + if strings.Contains(got, "nurse") { + t.Fatalf("occupation crowded participation answer: %q", out.Answer) + } + if strings.Contains(got, "hiking") { + t.Fatalf("hobby leftover crowded participation answer: %q", out.Answer) + } + if strings.Contains(got, "courage") || strings.Contains(got, "transition") { + t.Fatalf("transition leftover crowded participation answer: %q", out.Answer) + } + if strings.Contains(got, "adoption") { + t.Fatalf("mentor-contact leftover crowded participation answer: %q", out.Answer) + } +} + +func TestLooksParticipationSetQueryAndObjects(t *testing.T) { + q := "In what ways is Caroline participating in the LGBTQ community?" + if !looksParticipationSetQuery(q) { + t.Fatal("expected participation set") + } + if looksParticipationSetQuery("Which community activities have Riley and Casey participated in?") { + t.Fatal("dual community activity list must not use participation recover") + } + if looksParticipationSetQuery("What events is Maria planning for the homeless shelter funraiser?") { + t.Fatal("event leftover must not look like participation sets") + } + if looksParticipationSetQuery("In what ways did Caroline's life change after she started her transition?") { + t.Fatal("transition leftover must not look like participation sets") + } + got := strings.ToLower(strings.Join(participationObjectsFromContent("Caroline joined an LGBTQ activist group on 2023-07-18."), " ")) + if !strings.Contains(got, "activist") { + t.Fatalf("activist group=%q", got) + } + pride := strings.ToLower(strings.Join(participationObjectsFromContent("Caroline attended an LGBTQ+ pride parade on 2023-06-30."), " ")) + if !strings.Contains(pride, "pride") { + t.Fatalf("pride parade=%q", pride) + } + art := strings.ToLower(strings.Join(participationObjectsFromContent("Caroline is organizing an LGBTQ art show scheduled for September 2023."), " ")) + if !strings.Contains(art, "art") { + t.Fatalf("art show=%q", art) + } + ment := strings.ToLower(strings.Join(participationObjectsFromContent("Caroline joined a mentorship program for LGBTQ youth on the weekend of 15–16 July 2023."), " ")) + if !strings.Contains(ment, "mentorship") { + t.Fatalf("mentorship=%q", ment) + } + if got := participationObjectsFromContent("Caroline participates in hiking"); len(got) != 0 { + t.Fatalf("hobby participate-in leaked=%#v", got) + } + if got := participationObjectsFromContent("Caroline contacted her mentor for adoption advice."); len(got) != 0 { + t.Fatalf("mentor-contact leaked=%#v", got) + } +} + func TestRecallWhereKinshipPlaceNotActivityDump(t *testing.T) { t.Setenv("BRAINY_RECALL_LLM", "") store := newMemoryStoreStub() @@ -8911,6 +9037,17 @@ func TestLeftoverCoveringKeepsTypedItemJoins(t *testing.T) { if leftoverCoveringKeepTypedAnswer(benQ, benHops, csgo) { t.Fatal("CS:GO leftover must not count as a typed beneficiary join") } + partQ := "In what ways is Caroline participating in the LGBTQ community?" + partHops := []HopResult{{Kind: "fetch_predicate", Predicate: PredicateActivity, Entity: "Caroline", Source: "typed_store", ProofKind: "typed_exact", + Values: []string{"LGBTQ activist group", "LGBTQ+ pride parade", "LGBTQ art show", "mentorship program"}}} + partJoin := "LGBTQ activist group, LGBTQ+ pride parade, LGBTQ art show, mentorship program" + if !leftoverCoveringKeepTypedAnswer(partQ, partHops, partJoin) { + t.Fatal("typed participation join must be kept against leftover covering") + } + courage := "You've come a long way since your transition - keep on inspiring people with your strength and courage" + if leftoverCoveringKeepTypedAnswer(partQ, partHops, courage) { + t.Fatal("transition leftover must not count as a typed participation join") + } } func TestLeftoverCoveringWhereIgnoresHopSlotStarvation(t *testing.T) { From c02d70a62182f7ccce43701f0c2b08ac0703e6f9 Mon Sep 17 00:00:00 2001 From: Siddhant Singh Date: Wed, 26 Aug 2026 13:34:08 +0000 Subject: [PATCH 2/3] fix(recall): rank joined/organizing participation ahead of attended junk Compiler 'attended back' / 'attended not-so-great' rows were filling the activity-hop cap before organizing/host art-show objects. Primary cues now win the first slots; article-only participating-in still extracts 'show' lists without hobby 'participates in hiking'. Co-authored-by: aria --- internal/memory/hop_executor.go | 58 +++++++++++++++++++++++++-------- internal/memory/recall_test.go | 23 +++++++++++++ 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/internal/memory/hop_executor.go b/internal/memory/hop_executor.go index 2eadc84..8735f2c 100644 --- a/internal/memory/hop_executor.go +++ b/internal/memory/hop_executor.go @@ -1577,7 +1577,8 @@ func looksThinParticipation(v string) bool { switch v { case "community", "rights", "difference", "painting", "sidewalk", "month", "city", "neighborhood", "students", "folks", "adoption", "freedom", - "pride", "work", "people", "group", "events", "campaigns", "meetings": + "pride", "work", "people", "group", "events", "campaigns", "meetings", + "back", "those", "chasing", "not-so-great", "not so great": return true } if strings.HasPrefix(v, "community ") && !strings.Contains(v, "garden") { @@ -1590,9 +1591,9 @@ func recoverParticipationSlots(person string, listed []MemoryRecord) []recovered if person == "" { return nil } - var out []recoveredSlot + var primary, attend []recoveredSlot seen := map[string]struct{}{} - add := func(sl recoveredSlot) { + add := func(dst *[]recoveredSlot, sl recoveredSlot) { val := strings.TrimSpace(sl.value) val = strings.Trim(val, ".,;: ") if val == "" || anaphoricSlotValue(val) || looksCodedSlotValue(val) || utf8Len(val) > 48 || looksThinParticipation(val) { @@ -1603,7 +1604,7 @@ func recoverParticipationSlots(person string, listed []MemoryRecord) []recovered return } seen[key] = struct{}{} - out = append(out, sl) + *dst = append(*dst, sl) } for _, rec := range listed { content := strings.TrimSpace(rec.Content) @@ -1614,17 +1615,48 @@ func recoverParticipationSlots(person string, listed []MemoryRecord) []recovered !strings.HasPrefix(strings.ToLower(content), strings.ToLower(person)+":") { continue } - for _, v := range participationObjectsFromContent(content) { - add(recoveredSlot{value: v, content: content, memID: rec.MemoryID}) + for _, v := range participationObjectsFromCues(content, participationPrimaryCues) { + add(&primary, recoveredSlot{value: v, content: content, memID: rec.MemoryID}) + } + for _, v := range participationObjectsFromCues(content, participationAttendCues) { + add(&attend, recoveredSlot{value: v, content: content, memID: rec.MemoryID}) } } + out := append(primary, attend...) if len(out) > 8 { out = out[:8] } return out } +var participationPrimaryCues = []string{ + " joined a ", " joined an ", " joined the ", " joined ", + " organizing an ", " organizing a ", " host an ", " host a ", " hosting an ", " hosting a ", + " participating in a ", " participating in an ", " participating in the ", + " participates in a ", " participates in an ", " participates in the ", + "mentorship program", +} + +var participationAttendCues = []string{ + " attended a ", " attended an ", " attended the ", " attended ", +} + func participationObjectsFromContent(content string) []string { + seen := map[string]struct{}{} + var out []string + for _, v := range append(participationObjectsFromCues(content, participationPrimaryCues), + participationObjectsFromCues(content, participationAttendCues)...) { + key := strings.ToLower(v) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, v) + } + return out +} + +func participationObjectsFromCues(content string, cues []string) []string { lower := strings.ToLower(content) var out []string seen := map[string]struct{}{} @@ -1634,7 +1666,10 @@ func participationObjectsFromContent(content string) []string { if j := strings.IndexAny(v, ".!?"); j >= 0 { v = strings.TrimSpace(v[:j]) } - for _, tail := range []string{" which ", " during ", " on ", " last ", " after ", " and we ", " for ", " since ", " scheduled "} { + for _, tail := range []string{ + " which ", " during ", " on ", " last ", " after ", " and we ", " and ", + " for ", " since ", " scheduled ", " featuring ", " in ", + } { if k := strings.Index(strings.ToLower(v), tail); k >= 3 { v = strings.TrimSpace(v[:k]) } @@ -1646,7 +1681,7 @@ func participationObjectsFromContent(content string) []string { v = strings.TrimPrefix(v, "an ") v = strings.TrimPrefix(v, "the ") v = strings.TrimPrefix(v, "new ") - if v == "" || utf8Len(v) < 4 || utf8Len(v) > 48 || looksThinParticipation(v) { + if v == "" || utf8Len(v) < 5 || utf8Len(v) > 48 || looksThinParticipation(v) { return } key := strings.ToLower(v) @@ -1656,12 +1691,7 @@ func participationObjectsFromContent(content string) []string { seen[key] = struct{}{} out = append(out, v) } - for _, cue := range []string{ - " joined a ", " joined an ", " joined the ", " joined ", - " attended a ", " attended an ", " attended the ", " attended ", - " organizing an ", " organizing a ", " host an ", " host a ", " hosting an ", " hosting a ", - "mentorship program", - } { + for _, cue := range cues { start := 0 for { i := strings.Index(lower[start:], cue) diff --git a/internal/memory/recall_test.go b/internal/memory/recall_test.go index d498d47..734e94d 100644 --- a/internal/memory/recall_test.go +++ b/internal/memory/recall_test.go @@ -2537,6 +2537,20 @@ func TestRecallParticipationSetFromJoinedActivities(t *testing.T) { Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "contacted mentor for adoption advice", "subject": "Caroline"}, Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "contacted mentor for adoption advice", "subject": "Caroline"}, } + store.records["c-junk"] = MemoryRecord{ + MemoryID: "mem_cjunk", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline attended back", + DedupeKey: "cjunk", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "back", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "back", "subject": "Caroline"}, + } + store.records["c-nsg"] = MemoryRecord{ + MemoryID: "mem_cnsg", TenantID: "t-part", SubjectID: "u1", + Kind: KindFact, Content: "Caroline attended not-so-great", + DedupeKey: "cnsg", Status: StatusActive, UpdatedAt: now, + Metadata: map[string]any{"predicate": PredicateActivity, "value_norm": "not-so-great", "subject": "Caroline"}, + Explain: map[string]any{"predicate": PredicateActivity, "value_norm": "not-so-great", "subject": "Caroline"}, + } store.atoms = append(store.atoms, stubAtom{pred: PredicateAffiliation, val: "lgbtq activist group", memID: "mem_cact"}, stubAtom{pred: PredicateEvent, val: "pride parade", memID: "mem_cpride"}, @@ -2545,6 +2559,8 @@ func TestRecallParticipationSetFromJoinedActivities(t *testing.T) { stubAtom{pred: PredicateActivity, val: "hiking", memID: "mem_chike"}, stubAtom{pred: PredicateOccupation, val: "nurse", memID: "mem_cjob"}, stubAtom{pred: PredicateActivity, val: "contacted mentor for adoption advice", memID: "mem_cadv"}, + stubAtom{pred: PredicateActivity, val: "back", memID: "mem_cjunk"}, + stubAtom{pred: PredicateActivity, val: "not-so-great", memID: "mem_cnsg"}, ) out, err := svc.Recall(context.Background(), RecallRequest{ TenantID: "t-part", SubjectID: "u1", @@ -2597,6 +2613,10 @@ func TestLooksParticipationSetQueryAndObjects(t *testing.T) { if !strings.Contains(art, "art") { t.Fatalf("art show=%q", art) } + partArt := strings.ToLower(strings.Join(participationObjectsFromContent("Caroline is participating in an art show."), " ")) + if !strings.Contains(partArt, "art") { + t.Fatalf("participating art show=%q", partArt) + } ment := strings.ToLower(strings.Join(participationObjectsFromContent("Caroline joined a mentorship program for LGBTQ youth on the weekend of 15–16 July 2023."), " ")) if !strings.Contains(ment, "mentorship") { t.Fatalf("mentorship=%q", ment) @@ -2604,6 +2624,9 @@ func TestLooksParticipationSetQueryAndObjects(t *testing.T) { if got := participationObjectsFromContent("Caroline participates in hiking"); len(got) != 0 { t.Fatalf("hobby participate-in leaked=%#v", got) } + if got := participationObjectsFromContent("Caroline attended back"); len(got) != 0 { + t.Fatalf("attended-back leaked=%#v", got) + } if got := participationObjectsFromContent("Caroline contacted her mentor for adoption advice."); len(got) != 0 { t.Fatalf("mentor-contact leaked=%#v", got) } From cc96d1792fde9e367fa77c7b532ac145aa77ec5c Mon Sep 17 00:00:00 2001 From: Siddhant Singh Date: Wed, 26 Aug 2026 13:53:09 +0000 Subject: [PATCH 3/3] docs(benchmarks): pin P62 community participation sets 145/180 Same skip-ingest 180 as P61. Unique +1/-0: conv-26-q39 leftover courage slogan to joined/organizing/mentorship/pride set. Co-authored-by: aria --- docs/benchmarks/README.md | 1 + ...-mh-135-p62-product-recall-s1-1a5a7c.jsonl | 35 +++++++++ .../locomo-s0-diag-mh-135-p62-20260826.md | 55 ++++++++++++++ .../locomo-s0-diag-mh-135-p62-summary.json | 56 ++++++++++++++ docs/research/competitive/cycle-closeout.md | 74 +++++++++++++++++++ 5 files changed, 221 insertions(+) create mode 100644 docs/benchmarks/artifacts/failure-ledger/locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c.jsonl create mode 100644 docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-20260826.md create mode 100644 docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-summary.json diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index a68f168..8e51c12 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -196,6 +196,7 @@ Ladder: [research/public-bench-ladder.md](../research/public-bench-ladder.md). | P58 historical typed-set lists S0 this-VM (2026-08-26) | Same store, hybrid on. S2 product: scan past current-state for item/skill/place-set/`activities`+`done` questions; singular `location` stays a point fact; leftover how/why does not widen hop scan. **141/180** (MH **21/33**, OD **4/11**, SH **85/98**, temporal **31/38**). [pin](./artifacts/locomo-s0-diag-mh-135-p58-20260826.md). John outdoor hiking+mountaineering. Unique losses none vs P54. Collars/tricks still miss. **Not** leftover covering. **Not** n=1540, **not** a Mem0 same-pin. Not 90% (162/180 on this sample; n=1540 for public LoCoMo). | | P59 dest-class lists S0 this-VM (2026-08-26) | Same store, hybrid on. S2 product: dest-being skills without requiring `trick`; transfer-cue item objects (`buy`/`for`); possessed-class dest names (`dog named`, not generic `named`). **143/180** (MH **23/33**, OD **4/11**, SH **85/98**, temporal **31/38**). [pin](./artifacts/locomo-s0-diag-mh-135-p59-20260826.md). Audrey collars/tags/toys/beds; James swim/frisbee/skateboard. Unique losses none vs P58. p59/p59b 142/180 are not pins. **Not** leftover covering. **Not** n=1540, **not** a Mem0 same-pin. Not 90% (162/180 on this sample; n=1540 for public LoCoMo). | | P61 beneficiary org sets S0 this-VM (2026-08-26) | Same store, hybrid on. S2 product: who/which beneficiary questions recover raise/for-cue affiliation objects (shelter, homeless, hospital) instead of leftover tournament slogans. **144/180** (MH **24/33**, OD **4/11**, SH **85/98**, temporal **31/38**). [pin](./artifacts/locomo-s0-diag-mh-135-p61-20260826.md). Unique losses none vs P59. p60/p60b 143/180 food-set 180s are not pins. **Not** leftover covering. **Not** n=1540, **not** a Mem0 same-pin. Not 90% (162/180 on this sample; n=1540 for public LoCoMo). | +| P62 community participation sets S0 this-VM (2026-08-26) | Same store, hybrid on. S2 product: in-what-ways community questions recover joined/organizing/host/article-participating-in objects (activist group, art show, mentorship, pride event) instead of leftover courage slogans; attended compiler junk is ranked after. **145/180** (MH **25/33**, OD **4/11**, SH **85/98**, temporal **31/38**). [pin](./artifacts/locomo-s0-diag-mh-135-p62-20260826.md). Unique losses none vs P61. **Not** leftover covering. **Not** n=1540, **not** a Mem0 same-pin. Not 90% (162/180 on this sample; n=1540 for public LoCoMo). | | P42 how-did-start leftover covering S0 this-VM (2026-08-25) | Same store, hybrid on. How-did-start leftover covering admits duration-matched inception leftover on `how did … start … years ago`; covering prefers a multi-stem inception pair over a walking-only duration fact or a gym transformation/journey restatement; lexical search drops start/journey wrappers only on that query shape. **126/180** (MH **18/33**, OD **4/11**, SH **74/98**, temporal **30/38**). [pin](./artifacts/locomo-s0-diag-mh-135-p42-20260825.md). Diet+walking pair leftover recovered. Does not add a diet/walking or gym dictionary. **Not** n=1540, **not** a Mem0 same-pin. | | BEAM 100K this cycle | [beam-100k-fresh-20260815.md](./artifacts/beam-100k-fresh-20260815.md) | | BEAM 100K historical | [beam-100k-c0-async.md](./artifacts/beam-100k-c0-async.md) | diff --git a/docs/benchmarks/artifacts/failure-ledger/locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c.jsonl b/docs/benchmarks/artifacts/failure-ledger/locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c.jsonl new file mode 100644 index 0000000..7dd97bb --- /dev/null +++ b/docs/benchmarks/artifacts/failure-ledger/locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c.jsonl @@ -0,0 +1,35 @@ +{"ts": "2026-08-26T13:36:17Z", "dataset": "locomo-smoke", "question_id": "conv-26-q5", "question": "When did Melanie run a charity race?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "The sunday before 25 May 2023", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1280, "oracle_episode_count": 551, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 44}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "The sunday before 25 May 2023", "generated_answer": "Melanie ran the charity race on 20 May 2023."}, "notes": "The ground truth specifies the race was on the Sunday before 25 May 2023 (21 May 2023), but the predicted answer gives 20 May 2023, which is a different day."} +{"ts": "2026-08-26T13:36:31Z", "dataset": "locomo-smoke", "question_id": "conv-26-q30", "question": "Would Melanie be considered a member of the LGBTQ community?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Likely no, she does not refer to herself as part of it", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1237, "oracle_episode_count": 551, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "Likely no, she does not refer to herself as part of it", "generated_answer": "pottery, meeting activists, how art lets us get our emotions out, supported lgbtq youth"}, "notes": "The predicted answer does not state that Melanie is not a member of the LGBTQ community; it instead mentions unrelated topics and supporting LGBTQ youth, failing to match the ground truth."} +{"ts": "2026-08-26T13:37:01Z", "dataset": "locomo-smoke", "question_id": "conv-26-q65", "question": "What are some changes Caroline has faced during her transition journey?", "primary": "WRITE_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Changes to her body, losing unsupportive friends", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1280, "oracle_episode_count": 551, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 6}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "Changes to her body, losing unsupportive friends", "generated_answer": "Caroline gave a talk about her transgender journey at a school event on 2023-05-30."}, "notes": "The predicted answer mentions a talk Caroline gave, which does not include the key facts from the ground truth about body changes and losing unsupportive friends."} +{"ts": "2026-08-26T13:37:12Z", "dataset": "locomo-smoke", "question_id": "conv-26-q81", "question": "Would Caroline want to move back to her home country soon?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "No; she's in the process of adopting children.", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 24, "oracle_episode_count": 6, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1237, "oracle_episode_count": 551, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "No; she's in the process of adopting children.", "generated_answer": "Caroline's home country is Sweden."}, "notes": "The predicted answer only states Caroline's home country and does not convey the key fact that she does not want to move back because she is in the process of adopting children."} +{"ts": "2026-08-26T13:37:23Z", "dataset": "locomo-smoke", "question_id": "conv-26-q89", "question": "What does Melanie think about Caroline's decision to adopt?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "she thinks Caroline is doing something amazing and will be an awesome mom", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1237, "oracle_episode_count": 551, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 4}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "she thinks Caroline is doing something amazing and will be an awesome mom", "generated_answer": "pottery, important to cultivate a loving and accepting environment for her kids, spending time together and chilling at the beach, artist"}, "notes": "The predicted answer does not mention Melanie thinking Caroline's decision is amazing or that she will be an awesome mom; it instead talks about pottery, environment, beach, and being an artist, which are unrelated."} +{"ts": "2026-08-26T13:37:38Z", "dataset": "locomo-smoke", "question_id": "conv-26-q135", "question": "What setback did Melanie face in October 2023?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "She got hurt and had to take a break from pottery.", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1237, "oracle_episode_count": 551, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 12}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "She got hurt and had to take a break from pottery.", "generated_answer": "She was involved in a road\u2011trip accident that caused the car\u2019s airbags to deploy on the weekend of 14\u201315\u202fOctober\u202f2023."}, "notes": "The predicted answer mentions a road\u2011trip accident with airbags deploying, but does not state that Melanie was hurt and had to take a break from pottery, which are the essential facts in the ground truth."} +{"ts": "2026-08-26T13:39:06Z", "dataset": "locomo-smoke", "question_id": "conv-41-q145", "question": "How does John describe the camping trip with Max?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Peaceful and awesome", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1784, "oracle_episode_count": 787, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 7}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "Peaceful and awesome", "generated_answer": "John says he and Max \"had a blast\" on their camping trip last summer."}, "notes": "The predicted answer only conveys that the trip was fun ('had a blast'), which aligns with 'awesome' but does not mention or imply the 'peaceful' aspect of the trip, so it does not capture all key facts from the ground truth."} +{"ts": "2026-08-26T13:39:23Z", "dataset": "locomo-smoke", "question_id": "conv-42-q24", "question": "When is Nate hosting a gaming party?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "The weekend after 3June, 2022.", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1609, "oracle_episode_count": 657, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 74}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "The weekend after 3June, 2022.", "generated_answer": "Nate won his second gaming tournament on 30 April 2022."}, "notes": "The predicted answer states Nate won a tournament on 30 April 2022, which does not match the ground truth that Nate is hosting a gaming party on the weekend after 3 June 2022."} +{"ts": "2026-08-26T13:39:40Z", "dataset": "locomo-smoke", "question_id": "conv-42-q95", "question": "What is Nate's favorite video game?", "primary": "WRITE_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Xenoblade Chronicles", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 27, "oracle_episode_count": 3, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1598, "oracle_episode_count": 657, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 54}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "Xenoblade Chronicles", "generated_answer": "Loves Having Turtles, Turtles Bring Peace, Nintendo Games, Turtles, Holding Turtles, Catan"}, "notes": "The predicted answer does not mention or paraphrase 'Xenoblade Chronicles', which is the ground truth."} +{"ts": "2026-08-26T13:40:20Z", "dataset": "locomo-smoke", "question_id": "conv-43-q19", "question": "What is a prominent charity organization that John might want to work with and why?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Good Sports, because they work with Nike, Gatorade, and Under Armour and they aim toprovide youth sports opportunities for kids ages 3-18 in high-need communities.", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1973, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 1}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "Good Sports, because they work with Nike, Gatorade, and Under Armour and they aim toprovide youth sports opportunities for kids ages 3-18 in high-need communities.", "generated_answer": "John is teaming up with a local organization that helps disadvantaged kids with sports and school."}, "notes": "The predicted answer does not mention the specific charity Good Sports, its corporate partners, or the detailed mission of providing youth sports opportunities for ages 3-18 in high-need communities, which are essential facts from the ground truth."} +{"ts": "2026-08-26T13:40:34Z", "dataset": "locomo-smoke", "question_id": "conv-43-q48", "question": "When did John get an ankle injury in 2023?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "around November 16, 2023", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1998, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 38}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "around November 16, 2023", "generated_answer": "not in memory"}, "notes": "The predicted answer 'not in memory' does not provide the date of the ankle injury, which should be around November 16, 2023."} +{"ts": "2026-08-26T13:40:44Z", "dataset": "locomo-smoke", "question_id": "conv-43-q53", "question": "What other exercises can help John with his basketball performance?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Sprinting, long-distance running, and boxing.", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1973, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 9}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "Sprinting, long-distance running, and boxing.", "generated_answer": "Strength training, daily physical\u2011therapy exercises, jogging, and focused basketball shooting practice can all help John improve his performance."}, "notes": "The predicted answer lists strength training, physical\u2011therapy, jogging, and shooting practice, which do not match the ground\u2011truth exercises of sprinting, long\u2011distance running, and boxing."} +{"ts": "2026-08-26T13:41:13Z", "dataset": "locomo-smoke", "question_id": "conv-43-q113", "question": "Which basketball team does Tim support?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "The Wolves", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1973, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 13}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "The Wolves", "generated_answer": "John will play a basketball game in Seattle in August 2023. (15 August 2023)"}, "notes": "The predicted answer does not mention Tim's supported basketball team and is unrelated to the ground truth 'The Wolves'."} +{"ts": "2026-08-26T13:41:24Z", "dataset": "locomo-smoke", "question_id": "conv-43-q135", "question": "How does Tim stay motivated during difficult study sessions?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Visualizing goals and success", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1973, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 7}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "Visualizing goals and success", "generated_answer": "Tim breaks his studying into 25\u2011minute sessions with short breaks, takes mental breaks to let his mind wander and refresh, and leans on supportive people for encouragement."}, "notes": "The predicted answer mentions study breaks and support from others, but does not include the key fact that Tim stays motivated by visualizing his goals and success."} +{"ts": "2026-08-26T13:41:44Z", "dataset": "locomo-smoke", "question_id": "conv-43-q151", "question": "What is the topic of discussion between John and Tim on 11 December, 2023?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Academic achievements and sports successes", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1973, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "Academic achievements and sports successes", "generated_answer": "Tim maintains a good balance between studying and relaxation."}, "notes": "The predicted answer mentions balancing studying and relaxation, which does not match the ground truth topics of academic achievements and sports successes."} +{"ts": "2026-08-26T13:41:49Z", "dataset": "locomo-smoke", "question_id": "conv-43-q163", "question": "Which language is Tim learning?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "German", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1973, "oracle_episode_count": 734, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": true}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 12}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "German", "generated_answer": "Spanish"}, "notes": "The predicted answer 'Spanish' does not match the ground truth 'German'; the languages are different."} +{"ts": "2026-08-26T13:42:03Z", "dataset": "locomo-smoke", "question_id": "conv-44-q37", "question": "When did Audrey get into an accident in the park?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "between October 19 and 24, 2023", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 28, "oracle_episode_count": 2, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1763, "oracle_episode_count": 744, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 76}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "between October 19 and 24, 2023", "generated_answer": "Audrey hiked to a peak in a national park on 26 May 2023."}, "notes": "The predicted answer gives a date of 26 May 2023 and mentions a hike, which does not match the ground truth timeframe of between October 19 and 24, 2023 for the accident."} +{"ts": "2026-08-26T13:42:15Z", "dataset": "locomo-smoke", "question_id": "conv-44-q39", "question": "What did Audrey get wtih having so many dogs?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Companionship", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1741, "oracle_episode_count": 744, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": true}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 9}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "Companionship", "generated_answer": "She gets happiness and a sense of peace, feeling centered and recharged when her dogs are together."}, "notes": "The predicted answer mentions happiness, peace, and feeling centered, but does not state the key fact that Audrey gets companionship from having many dogs."} +{"ts": "2026-08-26T13:42:25Z", "dataset": "locomo-smoke", "question_id": "conv-44-q44", "question": "Which national park could Audrey and Andrew be referring to in their conversations?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Voyageurs National Park", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 28, "oracle_episode_count": 2, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1741, "oracle_episode_count": 744, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "Voyageurs National Park", "generated_answer": "great trail to hike on weekends; the views are awesome, standing on top of a mountain, adding"}, "notes": "The predicted answer does not mention Voyageurs National Park or any equivalent reference; it provides unrelated content about a trail and mountain views."} +{"ts": "2026-08-26T13:42:31Z", "dataset": "locomo-smoke", "question_id": "conv-44-q45", "question": "How many pets will Andrew have, as of December 2023?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "three", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1763, "oracle_episode_count": 744, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 13}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "three", "generated_answer": "4"}, "notes": "The predicted answer '4' does not match the ground truth answer 'three' (i.e., 3)."} +{"ts": "2026-08-26T13:42:55Z", "dataset": "locomo-smoke", "question_id": "conv-44-q62", "question": "What did Andrew express missing about exploring nature trails with his family's dog?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "The peaceful moments", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1741, "oracle_episode_count": 744, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "The peaceful moments", "generated_answer": "He said he misses exploring nature trails with his family's dog."}, "notes": "The ground truth specifies that Andrew missed 'the peaceful moments', whereas the predicted answer states he misses exploring nature trails with his family's dog, which does not capture the key fact about missing the peaceful moments."} +{"ts": "2026-08-26T13:43:33Z", "dataset": "locomo-smoke", "question_id": "conv-47-q33", "question": "Did John and James study together?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Yes", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1849, "oracle_episode_count": 729, "oracle_memory_count": 30, "oracle_representation_status": "complete"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": true}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "Yes", "generated_answer": "not in memory"}, "notes": "The predicted answer 'not in memory' does not state that John and James studied together, which contradicts the ground truth answer 'Yes'."} +{"ts": "2026-08-26T13:43:48Z", "dataset": "locomo-smoke", "question_id": "conv-47-q52", "question": "When did John start working on his 2D Adventure mobile game?", "primary": "WRITE_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "approximately summer of 2022", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1881, "oracle_episode_count": 729, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 15}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "approximately summer of 2022", "generated_answer": "John advanced to the next level in his video game during the week of 20 March 2022, which boosted his confidence."}, "notes": "The predicted answer mentions a March 2022 event unrelated to the start date of the 2D Adventure mobile game, and does not match the ground truth of approximately summer 2022."} +{"ts": "2026-08-26T13:43:56Z", "dataset": "locomo-smoke", "question_id": "conv-47-q53", "question": "How long did it take for James to complete his Witcher-inspired game?", "primary": "WRITE_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "six months", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1881, "oracle_episode_count": 729, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 16}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "six months", "generated_answer": "James finds checking off completed goals in his notebook very satisfying."}, "notes": "The predicted answer does not mention the duration of six months; it provides unrelated information about James's satisfaction with checking off goals."} +{"ts": "2026-08-26T13:44:43Z", "dataset": "locomo-smoke", "question_id": "conv-48-q50", "question": "Which year did Jolene start practicing yoga?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "2020", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1578, "oracle_episode_count": 704, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 8}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "temporal", "category_id": 2, "judgment": "WRONG", "ground_truth": "2020", "generated_answer": "Jolene has yoga mat blocks as of 28 March 2023."}, "notes": "The predicted answer does not provide the year Jolene started practicing yoga (2020) and instead gives unrelated information about yoga mat blocks in 2023."} +{"ts": "2026-08-26T13:44:55Z", "dataset": "locomo-smoke", "question_id": "conv-48-q77", "question": "Where did Jolene and her partner find a cool diving spot?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Phuket", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1578, "oracle_episode_count": 704, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": true}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 24}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "Phuket", "generated_answer": "not in memory"}, "notes": "The predicted answer 'not in memory' does not contain the key fact that the cool diving spot was in Phuket."} +{"ts": "2026-08-26T13:44:57Z", "dataset": "locomo-smoke", "question_id": "conv-48-q79", "question": "Has Deborah tried surfing?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "yes", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1578, "oracle_episode_count": 704, "oracle_memory_count": 30, "oracle_representation_status": "complete"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": true}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 43}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "yes", "generated_answer": "No, Deborah has not tried surfing yet."}, "notes": "The predicted answer states Deborah has not tried surfing, which contradicts the ground truth that she has tried surfing."} +{"ts": "2026-08-26T13:47:21Z", "dataset": "locomo-smoke", "question_id": "conv-48-q116", "question": "According to Jolene, what does exercise help her to feel?", "primary": "HARNESS_ERROR", "secondary": "", "flags": {"answer_ok": false, "gold": "connected to her body", "oracle_evidence": {"label": "HARNESS_ERROR", "answer_status": null, "explain": {}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "connected to her body", "generated_answer": "not in memory"}, "notes": "The predicted answer 'not in memory' does not include the key fact that exercise helps Jolene feel 'connected to her body', which is required."} +{"ts": "2026-08-26T13:47:50Z", "dataset": "locomo-smoke", "question_id": "conv-48-q155", "question": "What habits does Jolene practice to feel balanced?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "yoga, meditation, walks, and mindfulness", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1578, "oracle_episode_count": 704, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 44}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "yoga, meditation, walks, and mindfulness", "generated_answer": "Jolene balances herself by keeping a regular routine, practicing yoga and meditation, prioritizing self\u2011care, and listening to music during her practice."}, "notes": "The predicted answer mentions yoga and meditation but omits walks and mindfulness, which are part of the ground truth, and adds unrelated details."} +{"ts": "2026-08-26T13:48:02Z", "dataset": "locomo-smoke", "question_id": "conv-48-q172", "question": "What was one of Jolene's favorite games to play with her mom on the nintendo wii game system?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Monster Hunter: World", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1578, "oracle_episode_count": 704, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": true, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 0}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "Monster Hunter: World", "generated_answer": "My favorite is a park with a forest trail - it's so calming getting lost in nature"}, "notes": "The predicted answer does not mention or paraphrase the ground truth 'Monster Hunter: World' and is unrelated to the question."} +{"ts": "2026-08-26T13:48:24Z", "dataset": "locomo-smoke", "question_id": "conv-49-q18", "question": "What kind of healthy food suggestions has Evan given to Sam?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "flavored seltzer water, dark chocolate with high cocoa content, air-popped popcorn and fruit, veggies, healthy sandwich snacks, energy balls, grilled chicken salad with avocado", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1322, "oracle_episode_count": 576, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 6}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "flavored seltzer water, dark chocolate with high cocoa content, air-popped popcorn and fruit, veggies, healthy sandwich snacks, energy balls, grilled chicken salad with avocado", "generated_answer": "flavored seltzer water, dark chocolate, air-popped popcorn, fruit, energy balls, salad with chicken, avocado"}, "notes": "The predicted answer omits two key suggestions from the ground truth: veggies and healthy sandwich snacks."} +{"ts": "2026-08-26T13:48:37Z", "dataset": "locomo-smoke", "question_id": "conv-49-q37", "question": "What kind of healthy meals did Sam start eating after getting a health scare?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "salad, grilled salmon and vegetables, grilled chicken and veggie stir-fry, Beef Merlot, fruit bowl, smoothie bowl", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 28, "oracle_episode_count": 2, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1322, "oracle_episode_count": 576, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 2}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "salad, grilled salmon and vegetables, grilled chicken and veggie stir-fry, Beef Merlot, fruit bowl, smoothie bowl", "generated_answer": "grilled salmon with roasted vegetables, grilled chicken and veggie stir-fry"}, "notes": "The predicted answer only mentions grilled salmon with vegetables and grilled chicken stir-fry, omitting other meals listed in the ground truth such as salad, Beef Merlot, fruit bowl, and smoothie bowl."} +{"ts": "2026-08-26T13:48:55Z", "dataset": "locomo-smoke", "question_id": "conv-49-q71", "question": "Which major holiday season conincides with Evan's wedding?", "primary": "PROOF_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "Christmas", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 28, "oracle_episode_count": 2, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1302, "oracle_episode_count": 576, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_coverage": {"label": "PROOF_MISS", "answer_status": "supported", "explain": {"oracle_item_count": 17}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "open-domain", "category_id": 3, "judgment": "WRONG", "ground_truth": "Christmas", "generated_answer": "The photo of the homemade key lime pie comes from Sam's cousin's wedding."}, "notes": "The predicted answer does not mention Christmas or any major holiday season; it is unrelated to the question."} +{"ts": "2026-08-26T13:49:00Z", "dataset": "locomo-smoke", "question_id": "conv-49-q78", "question": "Who did Evan tell about his marriage?", "primary": "RETRIEVAL_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "To Sam, to his friends from work, and to his and his wife's families.", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "RETRIEVAL_MISS", "answer_status": "supported", "explain": {"oracle_fact_count": 29, "oracle_episode_count": 1, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1302, "oracle_episode_count": 576, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 2}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "multi-hop", "category_id": 1, "judgment": "WRONG", "ground_truth": "To Sam, to his friends from work, and to his and his wife's families.", "generated_answer": "He told his extended family."}, "notes": "The predicted answer only mentions telling his extended family, omitting the other key facts that he also told Sam and his friends from work, which are required by the ground truth."} +{"ts": "2026-08-26T13:50:04Z", "dataset": "locomo-smoke", "question_id": "conv-50-q113", "question": "What activity did Calvin enjoy during his summer drives?", "primary": "READER_MISS", "secondary": "", "flags": {"answer_ok": false, "gold": "feeling the wind blowing through his hair", "oracle_evidence": {"label": "", "answer_status": "supported", "explain": {"oracle_evidence_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_retrieval": {"label": "", "answer_status": "supported", "explain": {"oracle_fact_count": 30, "oracle_episode_count": 0, "oracle_memory_count": 30}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "oracle_representation": {"label": "", "answer_status": "supported", "explain": {"oracle_atom_count": 30, "oracle_fact_count": 1666, "oracle_episode_count": 732, "oracle_memory_count": 30, "oracle_representation_status": "partial"}, "gold_in_facts": false, "gold_in_facts_semantic": true, "gold_in_episodes": false}, "oracle_coverage": {"label": "", "answer_status": "supported", "explain": {"oracle_item_count": 46}, "gold_in_facts": false, "gold_in_facts_semantic": false, "gold_in_episodes": false}, "group": "single-hop", "category_id": 4, "judgment": "WRONG", "ground_truth": "feeling the wind blowing through his hair", "generated_answer": "He enjoyed an amazing drive (driving) on a summer day."}, "notes": "The predicted answer mentions enjoying a drive, but does not include the specific activity of feeling the wind blowing through his hair, which is the key fact in the ground truth."} diff --git a/docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-20260826.md b/docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-20260826.md new file mode 100644 index 0000000..a8be14b --- /dev/null +++ b/docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-20260826.md @@ -0,0 +1,55 @@ +# LoCoMo S0 product `/recall` — P62 community participation sets — 2026-08-26 + +Same frozen store as [locomo-s0-diag-mh-135-20260822.md](./locomo-s0-diag-mh-135-20260822.md): tenant `diag-mh-135` + conv-30, dataset SHA `79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4`, stratified 180 seed 1, `--fail-closed --skip-ingest`. Product SHA `c02d70a` (joined/organizing/host/article-participating-in recover onto the activity hop; attended compiler junk is ranked after). Hybrid **on** (`BRAINY_RECALL_LLM=1`). Go default for `BRAINY_RECALL_LLM` stays **off**. + +P61 pair: [locomo-s0-diag-mh-135-p61-20260826.md](./locomo-s0-diag-mh-135-p61-20260826.md) (`ee2baa6`, **144/180**). Honesty stop: [benchmax-audit-2026-08-25.md](../../research/competitive/benchmax-audit-2026-08-25.md). This is **S2 enumerate**, not leftover covering. + +**Not** n=1540. **Not** a Mem0 same-pin. **Not** SOTA. Does not replace integrity 32/180. Does not replace the reader-off 19/180 no-LLM pin. Does not replace README 11.4% / 70% 1×30. 90% on this 180 is **162/180**. 90% on public LoCoMo is n=1540. + +## Scores vs prior pins on this store + +| Lane | Overall | multi-hop | open-domain | single-hop | temporal | +| --- | ---: | ---: | ---: | ---: | ---: | +| product reader **off** (`453a929`) | **19/180 (0.106)** | **12/33** | 0/11 | 5/98 | 2/38 | +| product hybrid **on** P28 (`454fbb3`) | **113/180 (0.628)** | **18/33** | **4/11** | **61/98** | **30/38** | +| product hybrid **on** P53 (`ae15e40`) | **137/180 (0.761)** | **18/33** | **4/11** | **85/98** | **30/38** | +| product hybrid **on** P54 (`7653135`) | **140/180 (0.778)** | **20/33** | **4/11** | **85/98** | **31/38** | +| product hybrid **on** P58 (`4817e11`) | **141/180 (0.783)** | **21/33** | **4/11** | **85/98** | **31/38** | +| product hybrid **on** P59 (`2111b3b`) | **143/180 (0.794)** | **23/33** | **4/11** | **85/98** | **31/38** | +| product hybrid **on** P61 (`ee2baa6`) | **144/180 (0.800)** | **24/33** | **4/11** | **85/98** | **31/38** | +| product hybrid **on** P62 (`c02d70a`) | **145/180 (0.806)** | **25/33** | **4/11** | **85/98** | **31/38** | +| industry search+harness (reader-off pin) | 62/180 (0.344) | 10/33 | 3/11 | 27/98 | 22/38 | + +MH **24→25**. OD **held 4**. SH **held 85**. Temporal **held 31**. Product overall still leads this-VM industry 62/180 on the labeled product lane — still not a Mem0 same-pin. + +Item flips vs P61: **+1 / −0 = net +1**. Unique losses: **none**. + +Named recovery (generic S2, not a LoCoMo-named rule): + +- `conv-26-q39` Caroline community participation: leftover transition-courage line → **mentorship program, LGBTQ art show, LGBTQ activist group, LGBT pride event** (joined/organizing/host/article-participating-in objects, then attended; gold activist / pride parades / art show / mentoring accepted as paraphrase). + +Held: P61 beneficiary join (dog shelter, homeless, children's hospital); P59 dest-class (Audrey collars/tags/toys/beds; James swim/frisbee/skateboard); leftover covering (soda and candy); Jolene snakes Susie+Seraphim; Maria dogs Coco+Shadow; food-set joins (q18 suggestion list; q37 meals without pie). + +Still miss (next generic product, not covering): who-told marriage (`conv-49-q78`); food-set completeness vs WRITE-missing gold; OD 4/11; WRITE 4; polar surfing; Phuket where; companionship leftover (do not cover). + +## Failure ledger (35 misses) + +| Primary | P61 | P62 | +| --- | ---: | ---: | +| RETRIEVAL_MISS | 10 | 9 | +| PROOF_MISS | 14 | 14 | +| READER_MISS | 7 | 7 | +| WRITE_MISS | 4 | 4 | +| HARNESS_ERROR | 1 | 1 | + +Unique leftover losses remain **none**. RETRIEVAL 10→9 is q39 flipping off the leftover courage line. + +Largest P62 cells: `single-hop:PROOF_MISS` 8, `open-domain:PROOF_MISS` 5, `multi-hop:READER_MISS` 4, `temporal:RETRIEVAL_MISS` 3, `multi-hop:RETRIEVAL_MISS` 3. WRITE 4 — do not merge #133. + +## What this says + +1. "In what ways … community" questions were an S2 enumerate bug: leftover covering answered with a recency-top courage slogan while joined/attended/organizing/mentorship facts were already in the frozen store. Recovering those objects onto an activity hop, ranking joined/organizing/host ahead of attended compiler junk (`attended back`, `attended not-so-great`), and keeping the join against leftover covering recovers that set. +2. 145/180 is 80.6% on this sample, still far from 90% (**162/180**). Remaining mass is who-told lists, incomplete food gold (WRITE), OD hypotheticals, relative dates, and polar/where misses. Do not add leftover covering for companionship or peaceful moments. +3. Do not add an LGBTQ/activist/parade gold dictionary. Cues are joined / organizing / host / article participating-in / mentorship program / attended. + +Report: `locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c` (summary JSON + failure ledger in this folder). Auto smoke JSON/md dumps are not committed (secret scanner). diff --git a/docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-summary.json b/docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-summary.json new file mode 100644 index 0000000..8e90e68 --- /dev/null +++ b/docs/benchmarks/artifacts/locomo-s0-diag-mh-135-p62-summary.json @@ -0,0 +1,56 @@ +{ + "tenant_prefix": "diag-mh-135", + "runs": [ + { + "lane": "product-recall", + "run_id": "locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c", + "accuracy": 0.8055555555555556, + "correct": 145, + "total": 180, + "by_group": { + "temporal": { + "correct": 31, + "total": 38 + }, + "multi-hop": { + "correct": 25, + "total": 33 + }, + "open-domain": { + "correct": 4, + "total": 11 + }, + "single-hop": { + "correct": 85, + "total": 98 + } + }, + "failure_histogram": { + "total": 35, + "by_primary": { + "PROOF_MISS": 14, + "RETRIEVAL_MISS": 9, + "READER_MISS": 7, + "WRITE_MISS": 4, + "HARNESS_ERROR": 1 + }, + "by_group": { + "single-hop:PROOF_MISS": 8, + "open-domain:PROOF_MISS": 5, + "multi-hop:READER_MISS": 4, + "temporal:RETRIEVAL_MISS": 3, + "multi-hop:RETRIEVAL_MISS": 3, + "open-domain:RETRIEVAL_MISS": 2, + "temporal:WRITE_MISS": 2, + "single-hop:READER_MISS": 2, + "temporal:READER_MISS": 1, + "multi-hop:WRITE_MISS": 1, + "single-hop:RETRIEVAL_MISS": 1, + "single-hop:WRITE_MISS": 1, + "temporal:PROOF_MISS": 1, + "single-hop:HARNESS_ERROR": 1 + } + } + } + ] +} diff --git a/docs/research/competitive/cycle-closeout.md b/docs/research/competitive/cycle-closeout.md index 74bb14e..86b724c 100644 --- a/docs/research/competitive/cycle-closeout.md +++ b/docs/research/competitive/cycle-closeout.md @@ -5784,3 +5784,77 @@ Last Brainy pin **4/20**. Not re-run. **One step:** generic **S2 list completeness remainder** — community participation lists (`conv-26-q39`) and who-told lists (`conv-49-q78`) whose gold objects are in the frozen store. Do not fish food-set completeness against WRITE-missing sandwich snacks / Beef Merlot. Then **S2b OD** (still 4/11; 0/4 diagnostic). Then **S1 WRITE** with re-ingest. Then **S5 industry** (62/180 on this tenant). Do **not** queue leftover covering from this 180. Do not chase camping-peaceful. Do not invent Sunday. Do not steal across speakers. Do not special-case Scout. Do not add companionship covering. Isolated leftover covering is saturating. Remaining 36: RETRIEVAL 10, PROOF 14, READER 7, WRITE 4, HARNESS 1. Fair Mem0 180 waits on quota reset 2026-09-01. n=1540 only at S6. Do not merge #133. Do not write SOTA. Kill list unchanged. Start: [handover-sota-agent-2026-08-21.md](../handover-sota-agent-2026-08-21.md). +## 2026-08-26 — P62 community participation sets (145/180; not leftover covering) + +Pin: [locomo-s0-diag-mh-135-p62-20260826.md](../../benchmarks/artifacts/locomo-s0-diag-mh-135-p62-20260826.md). +Run `locomo-s0-diag-mh-135-p62-product-recall-s1-1a5a7c`. Product SHA **`c02d70a`**. +Honesty stop **`0c33eb8`** stays on `dev`; this increment is **S2 enumerate**, not covering. + +**90% honesty:** 90% on this 180 is **162/180**. 90% on public LoCoMo is **n=1540**, last pin **11.4%**. +This row is **145/180**. It is **not** 90%. It is **not** beating Mem0. + +### Landed + +Product: in-what-ways / ways + community questions enumerate activity objects recovered from listed memories with joined / organizing / host / article participating-in / mentorship-program cues, then attended. Compiler `attended back` / `attended not-so-great` rows are thin-stopped and ranked after primary slots so they cannot fill the hop cap before art-show/activist/mentorship objects. Dual-entity join stays off. Leftover covering keeps a ≥2 activity join. Named-community token filter still runs. **No** LoCoMo-named rules. **No** leftover-covering detector. **No** LGBTQ/activist/parade gold dictionary. + +Pin docs: this section; [docs/benchmarks/README.md](../../benchmarks/README.md) P62 row. + +**Do not merge** leftover-covering PRs **#133**, **#131**, **#143**, **#145**. + +### Own pins + +Same 180, seed 1, 10 convos, fail-closed skip-ingest, tenant `diag-mh-135`. Hybrid on. Dataset SHA `79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4`. + +| Pin | SHA | Overall | MH | OD | SH | temporal | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Product reader off | `453a929` | **19/180** | **12/33** | 0/11 | 5/98 | 2/38 | +| P28 | `454fbb3` | **113/180** | 18/33 | 4/11 | 61/98 | 30/38 | +| P53 (covering) | `ae15e40` | **137/180** | 18/33 | 4/11 | 85/98 | 30/38 | +| P54 (entity-scoped counts) | `7653135` | **140/180 (0.778)** | **20/33** | **4/11** | **85/98** | **31/38** | +| P58 (historical typed-set lists) | `4817e11` | **141/180 (0.783)** | **21/33** | **4/11** | **85/98** | **31/38** | +| P59 (dest-class lists) | `2111b3b` | **143/180 (0.794)** | **23/33** | **4/11** | **85/98** | **31/38** | +| P61 (beneficiary org sets) | `ee2baa6` | **144/180 (0.800)** | **24/33** | **4/11** | **85/98** | **31/38** | +| **P62 (community participation sets)** | **`c02d70a`** | **145/180 (0.806)** | **25/33** | **4/11** | **85/98** | **31/38** | +| Industry search+harness | same tenant | **62/180** | 10/33 | 3/11 | 27/98 | 22/38 | +| Full n=1540 product `/recall` | `1b5ab3e` | **175/1540 = 11.4%** | 7.4% MH | 5.2% OD | 10.5% SH | 19.0% temporal | +| 1×30 conv-26 | `1b5ab3e` | **21/30 (70%)** | 10/10 | **0/4** | — | 11/16 | +| LME-20 | `1b5ab3e` | **4/20** | | | | | + +Unique losses vs P61: **none**. Gain: Caroline community participation (`conv-26-q39`). Industry **62/180**. n=1540 and 1×30 **not** re-run; do not replace README 11.4% / 70%. + +### Competitor compare (detailed) + +#### 1. LoCoMo — trail (open-domain); this 180 is not public LoCoMo + +Public LoCoMo is **n=1540 / 11.4%**. This 180 is a stratified diagnostic. **OD 4/11** (0/4 diagnostic still). Do **not** write lead from 145/180 vs Mem0 11/30 (handicapped, different pin). Published Mem0 **92.5%** (their harness, top-k 200, n=1540) is **context**, never a scoreboard row. Fair Mem0 180 waits on quota **2026-09-01**. + +**Trailing axis (open-domain):** product mechanism still missing is **S2b** membership/hypothesis + **S1** write coverage (Xenoblade, yoga 2020, Phuket, Wolves). PoR: compiler coverage then re-ingest; not leftover covering. + +**Leading axes (must not regress):** OpMem 13/13, marketing 17/17, 1×30 MH 10/10. P61 beneficiary join, P59 dest-class lists, P54 counts, and P53 covering holds held on live `/recall`. + +#### 2. OpMem — lead (stale pin) + +Last **13/13**. Last Mem0 Platform ops pin **10/13**. **Lead ops.** Not re-run this increment. + +#### 3. Marketing vertical — lead (stale pin) + +Last **17/17** vs Mem0 empirical **4/17**. **Lead governed vertical.** + +#### 4. LME-20 — no pin this cycle + +Last Brainy pin **4/20**. Not re-run. + +#### 5. Graphiti / Zep — no pin + +**No same-pin.** + +**Mem0 OSS** was not re-measured. Platform fair 180 is **quota-blocked** until 2026-09-01. + +### Why + +`"In what ways is Caroline participating in the LGBTQ community?"` recovered a recency-top courage slogan because leftover covering beat an empty/wrong typed list. The four gold classes were already in listed memories as joined activist group, attended pride, organizing/hosting an art show, and a mentorship program. Recovering those objects onto an activity hop, ranking joined/organizing/host ahead of attended compiler junk, and keeping the join against leftover covering is the product mechanism. + +### Next + +**One step:** generic **S2 list completeness remainder** — who-told lists (`conv-49-q78`) whose gold objects are partly in the frozen store (work friends + extended family; do not invent Sam-as-told from in-dialogue). Do not fish food-set completeness against WRITE-missing sandwich snacks / Beef Merlot. Then **S2b OD** (still 4/11; 0/4 diagnostic). Then **S1 WRITE** with re-ingest. Then **S5 industry** (62/180 on this tenant). Do **not** queue leftover covering from this 180. Do not chase camping-peaceful. Do not invent Sunday. Do not steal across speakers. Do not special-case Scout. Do not add companionship covering. Isolated leftover covering is saturating. Remaining 35: RETRIEVAL 9, PROOF 14, READER 7, WRITE 4, HARNESS 1. Fair Mem0 180 waits on quota reset 2026-09-01. n=1540 only at S6. Do not merge #133. Do not write SOTA. Kill list unchanged. Start: [handover-sota-agent-2026-08-21.md](../handover-sota-agent-2026-08-21.md). +