Generated with Hive: Add epoch-ms magnitude discriminator helper and fix delta-filter to support mixed timestamp formats

mcp/src/graph/delta-since.integration.test.ts

@@ -0,0 +1,173 @@
+/**
+ * Integration test for the `$since` delta filter in `listQueryForLabel()`
+ * (exercised through the real read path: db.nodes_by_type, as used by the
+ * `GET /graph` endpoint).
+ *
+ * Seeds one legacy-seconds node (7-decimal string — the old Rust ingest
+ * format) and one new epoch-ms Integer node (what nowEpochMs() writes), then
+ * asserts:
+ *   1. a millisecond `$since` cursor older than both returns BOTH (the
+ *      regression: the old `toFloat(...) >= $since` comparison silently
+ *      dropped every legacy-seconds node once the frontend sent ms cursors);
+ *   2. a ms cursor between the two returns only the ms node (legacy seconds
+ *      must not over-match a newer ms cursor);
+ *   3. mixed-format nodes sort by *normalized* ms (ORDER BY);
+ *   4. returned Integer timestamps are coerced to plain numbers (no
+ *      `{low, high}` leak).
+ *
+ * Runs only against a live Neo4j at bolt://${NEO4J_HOST} (defaults
+ * localhost:7687 / neo4j / testtest, matching createNeo4jDriver). Skips when
+ * NO_DB=true or when unreachable, so `npm run test:node` stays DB-free.
+ * Run standalone with:
+ *   npx tsx --test --test-timeout=60000 src/graph/delta-since.integration.test.ts
+ */
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import neo4j from "neo4j-driver";
+
+import { db } from "./neo4j.js";
+import { toReturnNode } from "./utils.js";
+
+const noDb = process.env.NO_DB === "true" || process.env.NO_DB === "1";
+const host = process.env.NEO4J_HOST || "localhost:7687";
+const user = process.env.NEO4J_USER || "neo4j";
+const pswd = process.env.NEO4J_PASSWORD || "testtest";
+const uri = `bolt://${host}`;
+
+// --- reachability probe -----------------------------------------------------
+let reachable = false;
+const probeDriver = neo4j.driver(uri, neo4j.auth.basic(user, pswd), {
+  connectionTimeout: 3000,
+});
+try {
+  if (!noDb) {
+    await probeDriver.verifyConnectivity();
+    reachable = true;
+    console.log(`===> delta-since integration test using ${uri}`);
+  }
+} catch {
+  console.log(`===> Neo4j unreachable at ${uri} — skipping integration tests`);
+}
+await probeDriver.close().catch(() => {});
+
+const skipReason: string | false = noDb
+  ? "NO_DB=true"
+  : reachable
+    ? false
+    : `Neo4j unreachable at ${uri}`;
+
+// --- fixtures ---------------------------------------------------------------
+const NOW_MS = Date.now();
+const RUN = `t${NOW_MS}`;
+const LEGACY_KEY = `test-delta-legacy-${RUN}`;
+const MS_KEY = `test-delta-ms-${RUN}`;
+// legacy: epoch-seconds as a 7-decimal string (old Rust ingest format),
+// stored 2h ago
+const LEGACY_TS = ((NOW_MS - 2 * 3600_000) / 1000).toFixed(7);
+// new: epoch-ms Integer (nowEpochMs format), stored 1h ago
+const MS_TS = neo4j.int(NOW_MS - 1 * 3600_000);
+
+let driver: neo4j.Driver | null = null;
+async function run(cypher: string, params: Record<string, unknown> = {}) {
+  const session = driver!.session();
+  try {
+    return await session.run(cypher, params);
+  } finally {
+    await session.close();
+  }
+}
+
+before(async () => {
+  if (!reachable) return;
+  driver = neo4j.driver(uri, neo4j.auth.basic(user, pswd));
+  await run(
+    `MERGE (f:Data_Bank:Hint {node_key: $key})
+     ON CREATE SET f.ref_id = $ref_id, f.name = $name,
+                   f.date_added_to_graph = $ts`,
+    { key: LEGACY_KEY, ref_id: `ref-${LEGACY_KEY}`, name: "delta-legacy", ts: LEGACY_TS },
+  );
+  await run(
+    `MERGE (f:Data_Bank:Hint {node_key: $key})
+     ON CREATE SET f.ref_id = $ref_id, f.name = $name,
+                   f.date_added_to_graph = $ts`,
+    { key: MS_KEY, ref_id: `ref-${MS_KEY}`, name: "delta-ms", ts: MS_TS },
+  );
+});
+
+after(async () => {
+  if (!reachable) return;
+  await run(`MATCH (n) WHERE n.node_key IN [$lk, $mk] DETACH DELETE n`, {
+    lk: LEGACY_KEY,
+    mk: MS_KEY,
+  });
+  await driver?.close();
+});
+
+async function fetchNodes(sinceMs: number) {
+  return db.nodes_by_type("Hint", undefined, 50000, sinceMs);
+}
+
+describe("delta filter: ms $since over mixed legacy-seconds / new-ms stored values", () => {
+  it(
+    "a ms cursor older than both returns the legacy-seconds AND new-ms nodes",
+    { skip: skipReason },
+    async () => {
+      const nodes = await fetchNodes(NOW_MS - 3 * 3600_000);
+      const keys = new Set(nodes.map((n) => n.properties.node_key));
+      assert.ok(
+        keys.has(LEGACY_KEY),
+        "legacy-seconds node dropped by the ms $since filter",
+      );
+      assert.ok(keys.has(MS_KEY), "new-ms node dropped by the ms $since filter");
+    },
+  );
+
+  it(
+    "a ms cursor between the two returns only the new-ms node",
+    { skip: skipReason },
+    async () => {
+      const nodes = await fetchNodes(NOW_MS - 1.5 * 3600_000); // between -2h and -1h
+      const keys = new Set(nodes.map((n) => n.properties.node_key));
+      assert.ok(keys.has(MS_KEY), "ms node should match a cursor older than it");
+      assert.ok(
+        !keys.has(LEGACY_KEY),
+        "legacy node must not over-match a newer ms cursor",
+      );
+    },
+  );
+
+  it(
+    "orders mixed-format nodes by normalized ms (DESC)",
+    { skip: skipReason },
+    async () => {
+      const nodes = await fetchNodes(NOW_MS - 3 * 3600_000);
+      const msIdx = nodes.findIndex((n) => n.properties.node_key === MS_KEY);
+      const legacyIdx = nodes.findIndex(
+        (n) => n.properties.node_key === LEGACY_KEY,
+      );
+      assert.ok(msIdx !== -1 && legacyIdx !== -1);
+      assert.ok(
+        msIdx < legacyIdx,
+        `expected ms node (idx ${msIdx}) before legacy node (idx ${legacyIdx})`,
+      );
+    },
+  );
+
+  it(
+    "returned Integer timestamps are coerced to plain numbers (no {low, high} leak)",
+    { skip: skipReason },
+    async () => {
+      const nodes = await fetchNodes(NOW_MS - 3 * 3600_000);
+      const msNode = nodes.find((n) => n.properties.node_key === MS_KEY)!;
+      assert.ok(msNode, "ms node not found");
+      const ts = msNode.properties.date_added_to_graph as unknown;
+      assert.equal(typeof ts, "number", `expected number, got ${typeof ts}`);
+      assert.equal(ts, MS_TS.toNumber());
+      // The shaped API response (toReturnNode) must never carry a raw
+      // Integer object for this field. (Node.identity is also an Integer
+      // internally, but it is not part of the ReturnNode wire format.)
+      const json = JSON.stringify(toReturnNode(msNode));
+      assert.ok(!json.includes('"low"'), `leaked Integer object: ${json}`);
+    },
+  );
+});

mcp/src/graph/neo4j.ts

@@ -29,7 +29,7 @@ import {
   nameFileOnly,
 } from "./utils.js";
 import * as Q from "./queries.js";
-import { nowEpochMs, epochValueToMs } from "./time.js";
+import { nowEpochMs, toEpochMs, epochValueToMs } from "./time.js";
 import { vectorizeCodeDocument, vectorizeQuery } from "../vector/index.js";
 import { v4 as uuidv4 } from "uuid";
 import { createByModelName } from "@microsoft/tiktokenizer";
@@ -170,8 +170,10 @@ class Db {
     return results
       .flat()
       .sort((a, b) => {
-        const at = a.properties.date_added_to_graph || 0;
-        const bt = b.properties.date_added_to_graph || 0;
+        // Normalize mixed legacy-seconds/new-ms stored values before sorting
+        // (backfill shim — remove once the data migration has run).
+        const at = toEpochMs(a.properties.date_added_to_graph) ?? 0;
+        const bt = toEpochMs(b.properties.date_added_to_graph) ?? 0;
         return bt - at;
       })
       .slice(0, limit_total);

mcp/src/graph/queries.test.ts

@@ -1,37 +1,33 @@
 /**
- * Unit tests for `listQueryForLabel`'s `$since` filter after the
- * `toFloat(date_added_to_graph)` coercion shim removal.
+ * Unit tests for `listQueryForLabel`'s `$since` filter with mixed stored
+ * timestamp formats.
  *
- * `date_added_to_graph` is a canonical epoch-ms Integer (see ./time.ts), so
- * the property compares directly against an epoch-ms `$since`. These tests:
- *   1. pin the generated query text to the bare, coercion-free predicate;
- *   2. prove the clause's selection semantics on ms-magnitude fixtures —
- *      a recently-added node is selected, a stale one is not;
- *   3. prove a seconds-magnitude `$since` must be normalized (×1000) before
- *      binding — which `nodes_by_type` does via `epochValueToMs`.
+ * Stored `date_added_to_graph` values may be legacy epoch-seconds or new
+ * epoch-ms Integers until the data backfill migration. The generated query
+ * therefore normalizes the stored property via `epochMsExpr` (Cypher mirror
+ * of `toEpochMs`) before comparing against an epoch-ms `$since`.
+ *
+ * `nodes_by_type` also normalizes a seconds-magnitude caller `$since` via
+ * `epochValueToMs` before binding.
  *
  * Runs under NO_DB=true — no Neo4j contacted.
  */
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 
-import { listQueryForLabel } from "./queries.js";
-import { epochValueToMs } from "./time.js";
+import { epochMsExpr, listQueryForLabel } from "./queries.js";
+import { epochValueToMs, toEpochMs } from "./time.js";
 
 const HOUR_MS = 3600 * 1000;
 
-describe("listQueryForLabel $since filter (post toFloat removal)", () => {
-  it("compares date_added_to_graph directly — no toFloat coercion remains", () => {
+describe("listQueryForLabel $since filter (mixed timestamp formats)", () => {
+  it("normalizes stored date_added_to_graph via epochMsExpr before comparing", () => {
     const q = listQueryForLabel("Function", true);
-    assert.doesNotMatch(
-      q,
-      /toFloat\s*\(/,
-      "toFloat coercion shim must be gone"
-    );
-    assert.match(q, /f\.date_added_to_graph >= \$since/);
+    assert.match(q, /CASE WHEN toFloat\(f\.date_added_to_graph\) <= 1000000000000/);
+    assert.match(q, /f\.date_added_to_graph\) END >= \$since/);
     assert.match(
       q,
-      /ORDER BY coalesce\(f\.date_added_to_graph, 0\) DESC, f\.node_key/
+      /ORDER BY CASE WHEN toFloat\(coalesce\(f\.date_added_to_graph, 0\)\) <= 1000000000000/,
     );
   });
 
@@ -43,14 +39,14 @@ describe("listQueryForLabel $since filter (post toFloat removal)", () => {
   });
 
   /**
-   * Mirror of the generated clause:
-   *   `$since IS NULL OR (f.date_added_to_graph IS NOT NULL
-   *                       AND f.date_added_to_graph >= $since)`
-   * The text assertions above pin the template to this exact predicate, so
-   * the mirror cannot silently drift from what the query actually does.
+   * Mirror of the generated clause after both sides are normalized to ms:
+   *   `$since IS NULL OR (storedMs IS NOT NULL AND storedMs >= sinceMs)`
    */
   function selects(nodeDate: number | null, sinceMs: number | null): boolean {
-    return sinceMs === null || (nodeDate !== null && nodeDate >= sinceMs);
+    if (sinceMs === null) return true;
+    if (nodeDate === null) return false;
+    const storedMs = toEpochMs(nodeDate);
+    return storedMs !== null && storedMs >= sinceMs;
   }
 
   it("selects recently-added (ms-magnitude) nodes and excludes stale ones", () => {
@@ -75,6 +71,27 @@ describe("listQueryForLabel $since filter (post toFloat removal)", () => {
     );
   });
 
+  it("a millisecond $since cursor returns both legacy-seconds and new-ms nodes", () => {
+    const now = Date.now();
+    const sinceMs = now - 24 * HOUR_MS;
+    const legacySeconds = (now - 1 * HOUR_MS) / 1000; // 1h ago, stored as seconds
+    const newMs = now - 2 * HOUR_MS; // 2h ago, stored as ms
+    const tooOldSeconds = (now - 72 * HOUR_MS) / 1000;
+
+    assert.ok(
+      selects(legacySeconds, sinceMs),
+      "legacy-seconds node within the window must match a ms cursor"
+    );
+    assert.ok(
+      selects(newMs, sinceMs),
+      "new-ms node within the window must match the same cursor"
+    );
+    assert.ok(
+      !selects(tooOldSeconds, sinceMs),
+      "legacy-seconds node outside the window must still be excluded"
+    );
+  });
+
   it("normalizes a seconds-magnitude $since before comparing (as nodes_by_type does)", () => {
     const now = Date.now();
     const fresh = now - 1 * HOUR_MS;
@@ -98,4 +115,11 @@ describe("listQueryForLabel $since filter (post toFloat removal)", () => {
     assert.ok(selects(fresh, sinceMs), "fresh node still selected");
     assert.ok(!selects(stale, sinceMs), "stale node excluded again");
   });
+
+  it("epochMsExpr uses the same <= 1e12 boundary as toEpochMs", () => {
+    assert.equal(
+      epochMsExpr("f.date_added_to_graph"),
+      "CASE WHEN toFloat(f.date_added_to_graph) <= 1000000000000 THEN toFloat(f.date_added_to_graph) * 1000 ELSE toFloat(f.date_added_to_graph) END",
+    );
+  });
 });

mcp/src/graph/queries.ts

@@ -154,7 +154,7 @@ MATCH (l:Learning)
 OPTIONAL MATCH (l)-[:HAS_SCOPE]->(s:Scope)
 WITH l, collect(s.name) AS scopes
 RETURN l, scopes
-ORDER BY l.date_added_to_graph DESC
+ORDER BY ${epochMsExpr("l.date_added_to_graph")} DESC
 `;
 
 export const GET_ALL_SCOPES_QUERY = `
@@ -693,19 +693,35 @@ WHERE file.name ENDS WITH 'Cargo.toml'
 RETURN DISTINCT file
 `;
 
+/**
+ * Cypher mirror of `toEpochMs()` in `time.ts` — keep in sync. Normalizes a
+ * stored `date_added_to_graph` (legacy seconds float/string, or new epoch-ms
+ * Integer) to epoch milliseconds before comparing/sorting: values ≤ 1e12 are
+ * epoch-seconds (×1000), > 1e12 are already ms. `toFloat` handles the legacy
+ * 7-decimal strings, plain numbers, and Neo4j Integers alike. Load-bearing
+ * until the data backfill migration — without it a ms `$since` cursor would
+ * silently drop every legacy-seconds node.
+ *
+ * `nodes_by_type` also normalizes caller-supplied `since` values to ms via
+ * `epochValueToMs` before binding, so both sides of the comparison are ms.
+ */
+export function epochMsExpr(prop: string): string {
+  return `CASE WHEN toFloat(${prop}) <= 1000000000000 THEN toFloat(${prop}) * 1000 ELSE toFloat(${prop}) END`;
+}
+
 export function listQueryForLabel(
   label: string,
   withSince: boolean = false,
 ): string {
-  // `date_added_to_graph` is a canonical epoch-ms Integer (see ./time.ts), so
-  // it compares directly against an epoch-ms `$since` — no toFloat coercion.
-  // `nodes_by_type` normalizes caller-supplied `since` values to ms via
-  // `epochValueToMs` before binding.
   const sinceClause = withSince
-    ? `AND ($since IS NULL OR (f.date_added_to_graph IS NOT NULL AND f.date_added_to_graph >= $since))`
+    ? `AND ($since IS NULL OR (f.date_added_to_graph IS NOT NULL AND ${epochMsExpr(
+        "f.date_added_to_graph",
+      )} >= $since))`
     : "";
   const orderBy = withSince
-    ? `ORDER BY coalesce(f.date_added_to_graph, 0) DESC, f.node_key`
+    ? `ORDER BY ${epochMsExpr(
+        "coalesce(f.date_added_to_graph, 0)",
+      )} DESC, f.node_key`
     : "";
   return `
 MATCH (f:${label})