Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 89 additions & 30 deletions apps/memos-local-openclaw/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { fileURLToPath } from "url";
import { buildContext } from "./src/config";
import type { HostModelsConfig } from "./src/openclaw-api";
import { ensureSqliteBinding } from "./src/storage/ensure-binding";
import { rebuildBetterSqlite3 } from "./src/storage/rebuild-native";
import { SqliteStore } from "./src/storage/sqlite";
import { Embedder } from "./src/embedding";
import { IngestWorker } from "./src/ingest/worker";
Expand Down Expand Up @@ -166,7 +167,6 @@ const memosLocalPlugin = {

const moduleDir = path.dirname(fileURLToPath(import.meta.url));
const localRequire = createRequire(import.meta.url);
const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";

function detectPluginDir(startDir: string): string {
let cur = startDir;
Expand All @@ -193,40 +193,86 @@ const memosLocalPlugin = {
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
}

function runNpm(args: string[]) {
const { spawnSync } = localRequire("child_process") as typeof import("node:child_process");
return spawnSync(npmCmd, args, {
cwd: pluginDir,
stdio: "pipe",
shell: false,
timeout: 120_000,
});
}
/**
* Try to load better-sqlite3 from the plugin directory.
*
* Returns:
* - { ok: true }
* - { ok: false, reason: "missing" } — module not found
* - { ok: false, reason: "abi-mismatch" } — NODE_MODULE_VERSION mismatch (Issue #1734)
* - { ok: false, reason: "load-error" } — other native load failure
* - { ok: false, reason: "outside-plugin-dir" } — resolved to a path outside the plugin
*/
type SqliteLoadResult =
| { ok: true }
| { ok: false; reason: "missing" | "abi-mismatch" | "load-error" | "outside-plugin-dir"; message?: string };

function trySqliteLoad(): SqliteLoadResult {
let resolved: string;
try {
resolved = localRequire.resolve("better-sqlite3", { paths: [pluginDir] });
} catch (err) {
return { ok: false, reason: "missing", message: err instanceof Error ? err.message : String(err) };
}

let sqliteReady = false;
const resolvedReal = fs.existsSync(resolved) ? fs.realpathSync.native(resolved) : resolved;
if (!isPathInside(pluginDir, resolvedReal)) {
api.logger.warn(`memos-local: better-sqlite3 resolved outside plugin dir: ${resolved}`);
return { ok: false, reason: "outside-plugin-dir", message: resolved };
}

function trySqliteLoad(): boolean {
try {
const resolved = localRequire.resolve("better-sqlite3", { paths: [pluginDir] });
const resolvedReal = fs.existsSync(resolved) ? fs.realpathSync.native(resolved) : resolved;
if (!isPathInside(pluginDir, resolvedReal)) {
api.logger.warn(`memos-local: better-sqlite3 resolved outside plugin dir: ${resolved}`);
return false;
}
localRequire(resolvedReal);
return true;
} catch {
return false;
return { ok: true };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
// Native ABI mismatch — the compiled .node was built against a different
// Node.js NODE_MODULE_VERSION than the one currently running the Gateway.
// Issue #1734: this is the case we have to fix by rebuilding against the
// *current* Node binary, not whatever `npm` on PATH would pick up.
if (/NODE_MODULE_VERSION/.test(message)) {
return { ok: false, reason: "abi-mismatch", message };
}
return { ok: false, reason: "load-error", message };
}
}

sqliteReady = trySqliteLoad();
let loadResult = trySqliteLoad();
let sqliteReady = loadResult.ok;

// Helper that reads `reason` / `message` off a possibly-narrowed
// discriminated union without fighting TS narrowing through a mutable
// `let` binding.
const failureInfo = (
r: SqliteLoadResult,
): { reason: string; message: string } => {
if (r.ok) return { reason: "unknown", message: "" };
return { reason: r.reason, message: r.message ?? "" };
};

if (!sqliteReady) {
api.logger.warn(`memos-local: better-sqlite3 not found in ${pluginDir}, attempting auto-rebuild ...`);
const initial = failureInfo(loadResult);
const nodeAbi = process.versions.modules;
if (initial.reason === "abi-mismatch") {
api.logger.warn(
`memos-local: better-sqlite3 native binding was compiled against a different ` +
`Node.js ABI than the Gateway runtime (NODE_MODULE_VERSION=${nodeAbi}, Node ${process.version}). ` +
`Rebuilding with the Gateway's own Node binary (${process.execPath}) — this is the Issue #1734 fix path.`,
);
} else {
api.logger.warn(
`memos-local: better-sqlite3 not loadable from ${pluginDir} (reason=${initial.reason}), ` +
`attempting auto-rebuild pinned to Node ${process.version} (${process.execPath}) ...`,
);
}
if (initial.message) api.logger.warn(`memos-local: load error detail: ${initial.message.slice(0, 300)}`);

try {
const rebuildResult = runNpm(["rebuild", "better-sqlite3"]);
// CRITICAL (Issue #1734): rebuildBetterSqlite3 pins npm + node-gyp to
// process.execPath so the rebuild ABI always matches the current Node
// runtime — even when the user's `npm` on PATH points at a different
// Node installation (Homebrew vs nvm vs system).
const rebuildResult = rebuildBetterSqlite3({ pluginDir, timeoutMs: 120_000 });

const stdout = rebuildResult.stdout?.toString() || "";
const stderr = rebuildResult.stderr?.toString() || "";
Expand All @@ -237,11 +283,17 @@ const memosLocalPlugin = {
Object.keys(localRequire.cache)
.filter(k => k.includes("better-sqlite3") || k.includes("better_sqlite3"))
.forEach(k => delete localRequire.cache[k]);
sqliteReady = trySqliteLoad();
loadResult = trySqliteLoad();
sqliteReady = loadResult.ok;
if (sqliteReady) {
api.logger.info("memos-local: better-sqlite3 auto-rebuild succeeded!");
api.logger.info(
`memos-local: better-sqlite3 auto-rebuild succeeded (ABI ${nodeAbi}, Node ${process.version}).`,
);
} else {
api.logger.warn("memos-local: rebuild exited 0 but module still not loadable from plugin dir");
api.logger.warn(
`memos-local: rebuild exited 0 but module still not loadable ` +
`(reason=${failureInfo(loadResult).reason})`,
);
}
} else {
api.logger.warn(`memos-local: rebuild exited with code ${rebuildResult.status}`);
Expand All @@ -254,16 +306,20 @@ const memosLocalPlugin = {
const nodeVer = process.version;
const nodeMajor = parseInt(process.versions?.node?.split(".")[0] ?? "0", 10);
const isNode25Plus = nodeMajor >= 25;
const finalReason = failureInfo(loadResult).reason;
const lines = [
"",
"╔══════════════════════════════════════════════════════════════╗",
"║ MemOS Local Memory — better-sqlite3 native module missing ║",
"╠══════════════════════════════════════════════════════════════╣",
"║ ║",
"║ Auto-rebuild failed (Node " + nodeVer + "). Run manually: ║",
`║ Auto-rebuild failed (Node ${nodeVer}, reason=${finalReason}).${" ".repeat(Math.max(0, 12 - finalReason.length))}║`,
"║ Run manually with the SAME Node binary used by the ║",
"║ Gateway (this is what fixes Issue #1734): ║",
"║ ║",
`║ cd ${pluginDir}`,
"║ npm rebuild better-sqlite3 ║",
`║ ${process.execPath} \\`,
`║ $(command -v npm) rebuild better-sqlite3 ║`,
"║ openclaw gateway stop && openclaw gateway start ║",
"║ ║",
"║ If rebuild fails, install build tools first: ║",
Expand All @@ -280,7 +336,10 @@ const memosLocalPlugin = {
lines.push("");
api.logger.warn(lines.join("\n"));
throw new Error(
`better-sqlite3 native module not found (Node ${nodeVer}). Auto-rebuild failed. Fix: install build tools, then cd ${pluginDir} && npm rebuild better-sqlite3. Or use Node LTS (20/22).`
`better-sqlite3 native module not loadable under Node ${nodeVer} (reason=${finalReason}). ` +
`Auto-rebuild (pinned to ${process.execPath}) failed. ` +
`Fix: install build tools, then run \`${process.execPath} $(command -v npm) rebuild better-sqlite3\` in ${pluginDir}. ` +
`Or use Node LTS (20/22).`,
);
}
}
Expand Down
24 changes: 24 additions & 0 deletions apps/memos-local-openclaw/scripts/postinstall.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,28 @@ function phase(n, title) {
const pluginDir = path.resolve(__dirname, "..");
const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";

/**
* Build a child-process env that forces npm / node-gyp to use the *current*
* Node binary (`process.execPath`) instead of whatever `node` is found on
* PATH first. This is the postinstall-side mirror of the Issue #1734 fix
* applied in src/storage/rebuild-native.ts — keeping the two paths in sync
* means the binding always targets the same ABI no matter which entry
* point triggers the rebuild.
*/
function buildPinnedNodeEnv() {
const execDir = path.dirname(process.execPath);
const existingPath = process.env.PATH || process.env.Path || process.env.path || "";
const pinnedPath = existingPath
? execDir + path.delimiter + existingPath
: execDir;
return {
...process.env,
PATH: pinnedPath,
npm_node_execpath: process.execPath,
NODE: process.execPath,
};
}

function normalizePathForMatch(p) {
return path.resolve(p).replace(/^\\\\\?\\/, "").replace(/\\/g, "/").toLowerCase();
}
Expand Down Expand Up @@ -163,6 +185,7 @@ function ensureDependencies() {
stdio: "pipe",
shell: false,
timeout: 120_000,
env: buildPinnedNodeEnv(),
});
const elapsed = ((Date.now() - startMs) / 1000).toFixed(1);
const stderr = (result.stderr || "").toString().trim();
Expand Down Expand Up @@ -422,6 +445,7 @@ if (sqliteBindingsExist()) {
stdio: "pipe",
shell: false,
timeout: 180_000,
env: buildPinnedNodeEnv(),
});
const elapsed = ((Date.now() - startMs) / 1000).toFixed(1);
const stdout = (result.stdout || "").toString().trim();
Expand Down
27 changes: 17 additions & 10 deletions apps/memos-local-openclaw/src/storage/ensure-binding.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
import { existsSync, mkdirSync, copyFileSync } from "fs";
import { execSync } from "child_process";
import path from "path";
import { createRequire } from "module";
import { rebuildBetterSqlite3 } from "./rebuild-native";

/**
* Ensure the better-sqlite3 native binary is available.
*
* OpenClaw installs plugins with `--ignore-scripts`, which skips
* the native compilation step. This function checks for the binary
* and restores it from bundled prebuilds if missing.
*
* If we have to fall back to `npm rebuild`, the rebuild is pinned to
* `process.execPath` (the Gateway's current Node binary) via
* {@link rebuildBetterSqlite3} so it cannot pick up a different Node
* version from PATH and recreate the ABI mismatch (Issue #1734).
*/
export function ensureSqliteBinding(log?: { info: (msg: string) => void; warn: (msg: string) => void }): void {
const _req = typeof require !== "undefined" ? require : createRequire(__filename);
Expand All @@ -30,23 +35,25 @@ export function ensureSqliteBinding(log?: { info: (msg: string) => void; warn: (
return;
}

log?.warn(`[ensure-binding] No prebuild for ${platform}, attempting npm rebuild...`);
log?.warn(
`[ensure-binding] No prebuild for ${platform}, attempting npm rebuild ` +
`(pinned to Node ${process.version} at ${process.execPath})...`,
);
try {
const installDir = path.resolve(bsqlDir, "..", "..");
execSync("npm rebuild better-sqlite3", {
cwd: installDir,
stdio: "pipe",
timeout: 180_000,
});
if (existsSync(bindingPath)) {
const result = rebuildBetterSqlite3({ pluginDir: installDir });
if (result.status === 0 && existsSync(bindingPath)) {
log?.info(`[ensure-binding] Rebuilt better-sqlite3 successfully.`);
return;
}
if (result.status !== 0) {
log?.warn(`[ensure-binding] npm rebuild exited with code ${result.status}.`);
}
} catch { /* fall through */ }

throw new Error(
`better-sqlite3 native binary not found for ${platform}.\n` +
`Prebuild not bundled and npm rebuild failed.\n` +
`Fix: cd ${path.resolve(bsqlDir, "..", "..")} && npm rebuild better-sqlite3`,
`Prebuild not bundled and npm rebuild (under Node ${process.version}) failed.\n` +
`Fix: cd ${path.resolve(bsqlDir, "..", "..")} && ${process.execPath} $(command -v npm) rebuild better-sqlite3`,
);
}
82 changes: 82 additions & 0 deletions apps/memos-local-openclaw/src/storage/rebuild-native.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import * as path from "path";
import type { SpawnSyncOptions, SpawnSyncReturns } from "child_process";

/**
* Build the spawn environment used to rebuild `better-sqlite3` against the
* *current* Node binary (`process.execPath`).
*
* Issue #1734: when the OpenClaw Gateway runs under a different Node version
* than the one used at `npm install` time, `npm rebuild better-sqlite3`
* resolved through a plain PATH lookup may pick up yet another Node binary
* (Homebrew, nvm, system) and compile the binding against the wrong ABI,
* recreating the original `NODE_MODULE_VERSION` mismatch.
*
* The fix is to pin every Node-resolving lookup inside the child process to
* the binary that is actually loading the module right now:
*
* - `npm_node_execpath` / `NODE` are honored by npm and node-gyp to choose
* which Node executable (and matching headers) to use for the compile.
* - Prepending `dirname(process.execPath)` to `PATH` makes sure any shell
* script that calls `node` (npm wrappers, node-gyp helpers, …) finds the
* same binary first.
*/
export function buildRebuildEnv(
execPath: string = process.execPath,
baseEnv: NodeJS.ProcessEnv = process.env,
pathSeparator: string = path.delimiter,
): NodeJS.ProcessEnv {
const execDir = path.dirname(execPath);
const existingPath = baseEnv.PATH ?? baseEnv.Path ?? baseEnv.path ?? "";
const pinnedPath = existingPath
? `${execDir}${pathSeparator}${existingPath}`
: execDir;

return {
...baseEnv,
PATH: pinnedPath,
npm_node_execpath: execPath,
NODE: execPath,
};
}

export interface RebuildOptions {
/** Plugin directory passed as `cwd` to `npm rebuild better-sqlite3`. */
pluginDir: string;
/** Override the Node binary (defaults to `process.execPath`). */
execPath?: string;
/** Override `process.env` (used by tests). */
baseEnv?: NodeJS.ProcessEnv;
/** Inject `spawnSync` (used by tests). */
spawnSyncImpl?: (
command: string,
args: readonly string[],
options?: SpawnSyncOptions,
) => SpawnSyncReturns<Buffer>;
/** Timeout in milliseconds (default 180s, matches the previous behavior). */
timeoutMs?: number;
/** npm executable name (defaults to platform-appropriate). */
npmCmd?: string;
}

/**
* Run `npm rebuild better-sqlite3` with the pinned environment from
* {@link buildRebuildEnv}. Returns the raw `spawnSync` result so callers can
* inspect status/stdout/stderr.
*/
export function rebuildBetterSqlite3(opts: RebuildOptions): SpawnSyncReturns<Buffer> {
const execPath = opts.execPath ?? process.execPath;
const baseEnv = opts.baseEnv ?? process.env;
const npmCmd = opts.npmCmd ?? (process.platform === "win32" ? "npm.cmd" : "npm");
const env = buildRebuildEnv(execPath, baseEnv);

// eslint-disable-next-line @typescript-eslint/no-require-imports
const spawnSync = opts.spawnSyncImpl ?? (require("child_process") as typeof import("child_process")).spawnSync;

return spawnSync(npmCmd, ["rebuild", "better-sqlite3"], {
cwd: opts.pluginDir,
stdio: "pipe",
shell: false,
timeout: opts.timeoutMs ?? 180_000,
env,
});
}
Loading