Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,11 @@ jobs:
- name: Set up Node
uses: actions/setup-node@v6
with:
node-version: "20.19"
# The Vercel project runs Node.js 24.x. Node.js 20 gets no release
# without the compromised eMudhra root "emSign ECC Root CA - G3",
# and npm ci's audit and package-metadata requests are not checked
# against the lockfile's integrity hashes.
node-version: "24"
cache: npm
cache-dependency-path: frontend/package-lock.json

Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,11 @@ jobs:
- name: Set up Node
uses: actions/setup-node@v6
with:
node-version: "20.19"
# The Vercel project runs Node.js 24.x. Node.js 20 gets no release
# without the compromised eMudhra root "emSign ECC Root CA - G3",
# and npm ci's audit and package-metadata requests are not checked
# against the lockfile's integrity hashes.
node-version: "24"
cache: npm
cache-dependency-path: frontend/package-lock.json

Expand Down
21 changes: 21 additions & 0 deletions backend/tests/test_modal_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ def pip_install_from_requirements(self, *args):
def pip_install(self, *args):
return self._step("pip_install", *args)

def env(self, *args):
return self._step("env", *args)

def add_local_dir(self, *args, **kwargs):
return self._step("add_local_dir", *args, **kwargs)

Expand Down Expand Up @@ -99,8 +102,18 @@ def test_modal_deployment_definition_imports_without_remote_calls(monkeypatch):
"debian_slim",
"apt_install",
"pip_install_from_requirements",
"apt_install",
"pip_install",
"env",
"add_local_dir",
]
# The CA-list refresh follows the requirements step, so Modal keeps
# that step's cached build (see the comment in modal_app.py).
assert modal_app.image.steps[3][1] == ("ca-certificates",)
assert modal_app.image.steps[4][1] == ("certifi>=2026.10.19",)
assert modal_app.image.steps[5][1] == (
{"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": "/etc/ssl/certs/ca-certificates.crt"},
)
finally:
sys.modules.pop("modal_app", None)

Expand Down Expand Up @@ -140,6 +153,14 @@ def test_modal_eval_app_fans_out_twenty_cases_with_a_25_container_cap(
"create_if_missing": True,
"version": 2,
}
assert [step[0] for step in modal_eval_app.image.steps] == [
"debian_slim",
"pip_install_from_requirements",
"pip_install",
"add_local_dir",
"add_local_dir",
]
assert modal_eval_app.image.steps[2][1] == ("certifi>=2026.10.19",)
source = (REPO_ROOT / "modal_eval_app.py").read_text()
assert "evaluate_case.spawn_map(" in source
assert "concurrency=3" in source
Expand Down
10 changes: 10 additions & 0 deletions frontend/src/instrumentation.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Runs once per server instance, before it handles a request (Next.js 15
// instrumentation). It removes the compromised eMudhra roots from Node's CA
// list before the /api/proxy route calls the Modal backend; see
// src/utils/compromisedCaRoots.ts for when to delete this file.
export async function register() {
if (process.env.NEXT_RUNTIME === "nodejs") {
const { removeCompromisedEmudhraRoots } = await import("./utils/compromisedCaRoots");
removeCompromisedEmudhraRoots();
}
}
205 changes: 205 additions & 0 deletions frontend/src/utils/compromisedCaRoots.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
// @vitest-environment node
import { X509Certificate } from "node:crypto";
import { describe, expect, it, vi } from "vitest";

import {
COMPROMISED_ROOT_FINGERPRINTS,
removeCompromisedEmudhraRoots,
type CaCertificateApi,
} from "./compromisedCaRoots";

// The two roots the module removes, as Debian's ca-certificates 20230311
// ships them (emSign_ECC_Root_CA_-_G3.crt and Comodo_AAA_Services_root.crt).
const EMSIGN_ECC_ROOT_G3_PEM = `-----BEGIN CERTIFICATE-----
MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
+JbNR6iC8hZVdyR+EhCVBCyj
-----END CERTIFICATE-----
`;

const AAA_CERTIFICATE_SERVICES_PEM = `-----BEGIN CERTIFICATE-----
MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb
MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow
GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj
YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL
MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE
BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM
GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP
ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua
BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe
3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4
YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR
rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm
ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU
oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF
MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v
QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t
b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF
AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q
GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz
Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2
G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi
l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3
smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg==
-----END CERTIFICATE-----
`;

// An unrelated root the module must keep, as certifi 2026.10.23 ships it.
const ISRG_ROOT_X2_PEM = `-----BEGIN CERTIFICATE-----
MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw
CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg
R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00
MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT
ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw
EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW
+1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9
ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI
zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW
tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1
/q4AaOeMSQ+2b1tbFfLn
-----END CERTIFICATE-----
`;

// A stand-in for node:tls whose default CA list starts as `defaultCertificates`.
function createTlsStub(defaultCertificates: string[]) {
return {
getCACertificates: vi.fn((_type: "default") => [...defaultCertificates]),
setDefaultCACertificates: vi.fn((_certificates: string[]) => {}),
};
}

function createLoggerStub() {
return { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
}

describe("removeCompromisedEmudhraRoots", () => {
it("keys each root by the SHA-256 fingerprint of its PEM", () => {
expect(
COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(EMSIGN_ECC_ROOT_G3_PEM).fingerprint256),
).toBe("emSign ECC Root CA - G3");
expect(
COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(AAA_CERTIFICATE_SERVICES_PEM).fingerprint256),
).toBe("AAA Certificate Services");
expect(COMPROMISED_ROOT_FINGERPRINTS.size).toBe(2);
});

it("removes emSign ECC Root CA - G3 and keeps an unrelated root", () => {
const tlsStub = createTlsStub([ISRG_ROOT_X2_PEM, EMSIGN_ECC_ROOT_G3_PEM]);
const loggerStub = createLoggerStub();
removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub });

expect(tlsStub.getCACertificates).toHaveBeenCalledWith("default");
expect(tlsStub.setDefaultCACertificates).toHaveBeenCalledTimes(1);
expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]);
expect(loggerStub.info).toHaveBeenCalledTimes(1);
const infoMessage = loggerStub.info.mock.calls[0][0];
expect(infoMessage).toContain(
"Removed 1 certificate(s) of the compromised roots (emSign ECC Root CA - G3)",
);
expect(infoMessage).toContain("(2 -> 1 certificates)");
expect(infoMessage).toContain(`Node.js ${process.version}`);
expect(loggerStub.warn).not.toHaveBeenCalled();
expect(loggerStub.error).not.toHaveBeenCalled();
});

it("removes AAA Certificate Services, and every copy of a root listed more than once", () => {
const tlsStub = createTlsStub([
EMSIGN_ECC_ROOT_G3_PEM,
AAA_CERTIFICATE_SERVICES_PEM,
ISRG_ROOT_X2_PEM,
EMSIGN_ECC_ROOT_G3_PEM,
]);
const loggerStub = createLoggerStub();
removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub });

expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]);
expect(loggerStub.info.mock.calls[0][0]).toContain(
"Removed 3 certificate(s) of the compromised roots (emSign ECC Root CA - G3, AAA Certificate Services)",
);
});

it("sets a list without the roots unchanged and reports 0 removed", () => {
const tlsStub = createTlsStub([ISRG_ROOT_X2_PEM]);
const loggerStub = createLoggerStub();
removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub });

expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]);
const infoMessage = loggerStub.info.mock.calls[0][0];
expect(infoMessage).toContain("Removed 0 certificate(s) of the compromised roots from");
expect(infoMessage).toContain("(1 -> 1 certificates)");
});

it.each<[string, CaCertificateApi]>([
["tls.getCACertificates", { setDefaultCACertificates: vi.fn() }],
["tls.setDefaultCACertificates", { getCACertificates: vi.fn(() => [EMSIGN_ECC_ROOT_G3_PEM]) }],
])("without %s it warns and changes nothing", (_missingFunctionName, partialTlsStub) => {
const loggerStub = createLoggerStub();
removeCompromisedEmudhraRoots({ tls: partialTlsStub, logger: loggerStub });

for (const presentFunction of Object.values(partialTlsStub)) {
expect(presentFunction).not.toHaveBeenCalled();
}
expect(loggerStub.warn).toHaveBeenCalledTimes(1);
expect(loggerStub.warn.mock.calls[0][0]).toContain(
`Node.js ${process.version} has no tls.getCACertificates`,
);
expect(loggerStub.info).not.toHaveBeenCalled();
expect(loggerStub.error).not.toHaveBeenCalled();
});

it("logs an error from tls.getCACertificates with the error instead of throwing it", () => {
const readError = new Error("cannot read the CA list");
const tlsStub = createTlsStub([]);
tlsStub.getCACertificates.mockImplementation(() => {
throw readError;
});
const loggerStub = createLoggerStub();

expect(() => removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub })).not.toThrow();
expect(tlsStub.setDefaultCACertificates).not.toHaveBeenCalled();
expect(loggerStub.error).toHaveBeenCalledTimes(1);
const [errorMessage, loggedError] = loggerStub.error.mock.calls[0];
expect(errorMessage).toContain(
`Failed to remove the compromised roots from Node's default CA list on Node.js ${process.version}`,
);
expect(errorMessage).toContain("Routes keep serving.");
expect(loggedError).toBe(readError);
expect(loggerStub.info).not.toHaveBeenCalled();
});

it("logs an unparseable certificate or an error from tls.setDefaultCACertificates instead of throwing", () => {
const unparseableTlsStub = createTlsStub([
"-----BEGIN CERTIFICATE-----\nnot a certificate\n-----END CERTIFICATE-----\n",
]);
const unparseableLoggerStub = createLoggerStub();
expect(() =>
removeCompromisedEmudhraRoots({ tls: unparseableTlsStub, logger: unparseableLoggerStub }),
).not.toThrow();
expect(unparseableTlsStub.setDefaultCACertificates).not.toHaveBeenCalled();
expect(unparseableLoggerStub.error).toHaveBeenCalledTimes(1);
expect(unparseableLoggerStub.error.mock.calls[0][1]).toBeInstanceOf(Error);

const writeError = new TypeError("bad certificate list");
const rejectingTlsStub = createTlsStub([EMSIGN_ECC_ROOT_G3_PEM]);
rejectingTlsStub.setDefaultCACertificates.mockImplementation(() => {
throw writeError;
});
const rejectingLoggerStub = createLoggerStub();
expect(() =>
removeCompromisedEmudhraRoots({ tls: rejectingTlsStub, logger: rejectingLoggerStub }),
).not.toThrow();
expect(rejectingLoggerStub.error.mock.calls[0][1]).toBe(writeError);
expect(rejectingLoggerStub.info).not.toHaveBeenCalled();
});
});
109 changes: 109 additions & 0 deletions frontend/src/utils/compromisedCaRoots.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Server only: called from src/instrumentation.ts when a server instance
// starts, before the /api/proxy route forwards requests (with the user's
// Authorization header) to the Modal backend.
//
// Temporary. On 2026-10-20 it was disclosed that the private keys of eight
// eMudhra (emSign) intermediate CA certificates can be recovered
// (https://emudhra.we-have-a-problem.com). Six of them are TLS intermediates
// that chain to "emSign ECC Root CA - G3", and all eight are also cross-signed
// by Sectigo's "AAA Certificate Services". Vercel Functions ran Node.js
// 24.21.0 and 22.23.2 when last measured (2026-10-21), and a PolicyEngine
// preview function still ran 24.21.0 on 2026-10-26. The CA list compiled
// into both releases includes "emSign ECC Root CA - G3"; the one in Node.js
// 24.22.1 and 22.23.4 does not. This module removes both roots from
// Node's default CA list, which TLS connections opened after this call use
// (fetch included) unless they pass their own `ca` option.
//
// Delete this module, its test and src/instrumentation.ts once Vercel
// Functions run Node.js 24.22.1 / 22.23.4 or later. The info message below
// logs process.version each time a server instance starts, so the function
// logs show which version is running.
import { X509Certificate } from "node:crypto";
import nodeTls from "node:tls";

const LOG_PREFIX = "[compromisedCaRoots]";

// Root certificates to remove, keyed by SHA-256 fingerprint in the format of
// X509Certificate#fingerprint256. Each fingerprint matches the CCADB "All
// Certificate Records" report and the PEM in Debian's ca-certificates
// 20230311.
export const COMPROMISED_ROOT_FINGERPRINTS: ReadonlyMap<string, string> = new Map([
// emSign ECC Root CA - G3 (eMudhra). Also matches the PEM in the CA list
// compiled into Node.js 24.21.0 and 22.23.2.
[
"86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B",
"emSign ECC Root CA - G3",
],
// AAA Certificate Services (Sectigo, formerly Comodo). Removed only as a
// defensive measure: the CA list compiled into Node.js 24.21.0 and 22.23.2
// does not contain it, because Mozilla trusts it for email only and Node.js
// compiles in only the roots Mozilla trusts for websites. It can reach
// Node's default CA list through NODE_EXTRA_CA_CERTS, or through
// --use-system-ca (or NODE_USE_SYSTEM_CA=1) on an operating system whose CA
// store still contains it.
[
"D7:A7:A0:FB:5D:7E:27:31:D7:71:E9:48:4E:BC:DE:F7:1D:5F:0C:3E:0A:29:48:78:2B:C8:3E:E0:EA:69:9E:F4",
"AAA Certificate Services",
],
]);

// The parts of node:tls this module uses. Both functions are optional because
// Node.js releases before 22.19 / 24.5 lack setDefaultCACertificates.
export type CaCertificateApi = {
getCACertificates?: (type: "default") => string[];
setDefaultCACertificates?: (certificates: string[]) => void;
};

export type CaFilterLogger = Pick<Console, "info" | "warn" | "error">;

// `tls` and `logger` are parameters so the unit test can pass stubs.
//
// An error is logged, not rethrown. Next.js rethrows an error from
// instrumentation's register() while it prepares the server, so a throw here
// would make every function-served route fail, the backend proxy included.
// The filter changes only which roots the functions' outbound HTTPS requests
// trust, so keeping the routes available is preferred; the error message says
// the roots may still be trusted.
export function removeCompromisedEmudhraRoots({
tls = nodeTls,
logger = console,
}: { tls?: CaCertificateApi; logger?: CaFilterLogger } = {}): void {
if (typeof tls.getCACertificates !== "function" || typeof tls.setDefaultCACertificates !== "function") {
logger.warn(
`${LOG_PREFIX} Node.js ${process.version} has no tls.getCACertificates or ` +
"tls.setDefaultCACertificates; the compromised roots were not removed from Node's " +
"default CA list.",
);
return;
}
try {
const defaultCertificates = tls.getCACertificates("default");
// A root can appear more than once (an operating-system store added by
// --use-system-ca may list it several times), so count certificates and
// name each removed root once.
const removedRootNames = new Set<string>();
const keptCertificates = defaultCertificates.filter((pem) => {
const rootName = COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(pem).fingerprint256);
if (rootName === undefined) {
return true;
}
removedRootNames.add(rootName);
return false;
});
tls.setDefaultCACertificates(keptCertificates);
const removedCount = defaultCertificates.length - keptCertificates.length;
const removedNamesText = removedCount > 0 ? ` (${[...removedRootNames].join(", ")})` : "";
logger.info(
`${LOG_PREFIX} Removed ${removedCount} certificate(s) of the compromised roots` +
`${removedNamesText} from Node's default CA list (${defaultCertificates.length} -> ` +
`${keptCertificates.length} certificates) on Node.js ${process.version}. Delete this ` +
"module once Vercel Functions run Node.js 24.22.1 / 22.23.4 or later.",
);
} catch (error) {
logger.error(
`${LOG_PREFIX} Failed to remove the compromised roots from Node's default CA list on ` +
`Node.js ${process.version}; the list may still trust them. Routes keep serving.`,
error,
);
}
}
Loading
Loading