From 21fce75a7dd746d082fb55542ddcf4a0ffcc426a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:51:11 +0400 Subject: [PATCH 1/2] Stop trusting the compromised eMudhra CA certificates The private keys of eight eMudhra (emSign) intermediate CA certificates can be recovered from the public certificates (https://emudhra.we-have-a-problem.com). - Vercel Functions: remove "emSign ECC Root CA - G3" and, as a defensive measure, "AAA Certificate Services" from Node's default CA list when a server instance starts (src/instrumentation.ts), matched by SHA-256 fingerprint. Vercel Functions still run Node.js 24.21.0, whose compiled-in CA list includes the emSign root, and the /api/proxy route forwards Authorization headers to the Modal backend. - Modal backend image: after the requirements step, upgrade Debian's ca-certificates, require certifi>=2026.10.19, and point gRPC at the refreshed Debian bundle with GRPC_DEFAULT_SSL_ROOTS_FILE_PATH. - Modal eval image: require certifi>=2026.10.19 after the requirements step. Co-Authored-By: Claude Opus 5.5 --- backend/tests/test_modal_app.py | 21 ++ frontend/src/instrumentation.ts | 10 + frontend/src/utils/compromisedCaRoots.test.ts | 205 ++++++++++++++++++ frontend/src/utils/compromisedCaRoots.ts | 109 ++++++++++ modal_app.py | 18 ++ modal_eval_app.py | 6 + 6 files changed, 369 insertions(+) create mode 100644 frontend/src/instrumentation.ts create mode 100644 frontend/src/utils/compromisedCaRoots.test.ts create mode 100644 frontend/src/utils/compromisedCaRoots.ts diff --git a/backend/tests/test_modal_app.py b/backend/tests/test_modal_app.py index 2e6d77fe..bda680dd 100644 --- a/backend/tests/test_modal_app.py +++ b/backend/tests/test_modal_app.py @@ -49,6 +49,9 @@ def pip_install_from_requirements(self, *args): def pip_install(self, *args): return self._step("pip_install", *args) + def env(self, *args): + return self._step("env", *args) + def add_local_dir(self, *args, **kwargs): return self._step("add_local_dir", *args, **kwargs) @@ -99,8 +102,18 @@ def test_modal_deployment_definition_imports_without_remote_calls(monkeypatch): "debian_slim", "apt_install", "pip_install_from_requirements", + "apt_install", + "pip_install", + "env", "add_local_dir", ] + # The CA-list refresh follows the requirements step, so Modal keeps + # that step's cached build (see the comment in modal_app.py). + assert modal_app.image.steps[3][1] == ("ca-certificates",) + assert modal_app.image.steps[4][1] == ("certifi>=2026.10.19",) + assert modal_app.image.steps[5][1] == ( + {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": "/etc/ssl/certs/ca-certificates.crt"}, + ) finally: sys.modules.pop("modal_app", None) @@ -140,6 +153,14 @@ def test_modal_eval_app_fans_out_twenty_cases_with_a_25_container_cap( "create_if_missing": True, "version": 2, } + assert [step[0] for step in modal_eval_app.image.steps] == [ + "debian_slim", + "pip_install_from_requirements", + "pip_install", + "add_local_dir", + "add_local_dir", + ] + assert modal_eval_app.image.steps[2][1] == ("certifi>=2026.10.19",) source = (REPO_ROOT / "modal_eval_app.py").read_text() assert "evaluate_case.spawn_map(" in source assert "concurrency=3" in source diff --git a/frontend/src/instrumentation.ts b/frontend/src/instrumentation.ts new file mode 100644 index 00000000..5bc77a4a --- /dev/null +++ b/frontend/src/instrumentation.ts @@ -0,0 +1,10 @@ +// Runs once per server instance, before it handles a request (Next.js 15 +// instrumentation). It removes the compromised eMudhra roots from Node's CA +// list before the /api/proxy route calls the Modal backend; see +// src/utils/compromisedCaRoots.ts for when to delete this file. +export async function register() { + if (process.env.NEXT_RUNTIME === "nodejs") { + const { removeCompromisedEmudhraRoots } = await import("./utils/compromisedCaRoots"); + removeCompromisedEmudhraRoots(); + } +} diff --git a/frontend/src/utils/compromisedCaRoots.test.ts b/frontend/src/utils/compromisedCaRoots.test.ts new file mode 100644 index 00000000..2a9e8232 --- /dev/null +++ b/frontend/src/utils/compromisedCaRoots.test.ts @@ -0,0 +1,205 @@ +// @vitest-environment node +import { X509Certificate } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; + +import { + COMPROMISED_ROOT_FINGERPRINTS, + removeCompromisedEmudhraRoots, + type CaCertificateApi, +} from "./compromisedCaRoots"; + +// The two roots the module removes, as Debian's ca-certificates 20230311 +// ships them (emSign_ECC_Root_CA_-_G3.crt and Comodo_AAA_Services_root.crt). +const EMSIGN_ECC_ROOT_G3_PEM = `-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- +`; + +const AAA_CERTIFICATE_SERVICES_PEM = `-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- +`; + +// An unrelated root the module must keep, as certifi 2026.10.23 ships it. +const ISRG_ROOT_X2_PEM = `-----BEGIN CERTIFICATE----- +MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw +CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg +R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 +MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw +EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW ++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 +ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI +zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW +tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 +/q4AaOeMSQ+2b1tbFfLn +-----END CERTIFICATE----- +`; + +// A stand-in for node:tls whose default CA list starts as `defaultCertificates`. +function createTlsStub(defaultCertificates: string[]) { + return { + getCACertificates: vi.fn((_type: "default") => [...defaultCertificates]), + setDefaultCACertificates: vi.fn((_certificates: string[]) => {}), + }; +} + +function createLoggerStub() { + return { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; +} + +describe("removeCompromisedEmudhraRoots", () => { + it("keys each root by the SHA-256 fingerprint of its PEM", () => { + expect( + COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(EMSIGN_ECC_ROOT_G3_PEM).fingerprint256), + ).toBe("emSign ECC Root CA - G3"); + expect( + COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(AAA_CERTIFICATE_SERVICES_PEM).fingerprint256), + ).toBe("AAA Certificate Services"); + expect(COMPROMISED_ROOT_FINGERPRINTS.size).toBe(2); + }); + + it("removes emSign ECC Root CA - G3 and keeps an unrelated root", () => { + const tlsStub = createTlsStub([ISRG_ROOT_X2_PEM, EMSIGN_ECC_ROOT_G3_PEM]); + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub }); + + expect(tlsStub.getCACertificates).toHaveBeenCalledWith("default"); + expect(tlsStub.setDefaultCACertificates).toHaveBeenCalledTimes(1); + expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]); + expect(loggerStub.info).toHaveBeenCalledTimes(1); + const infoMessage = loggerStub.info.mock.calls[0][0]; + expect(infoMessage).toContain( + "Removed 1 certificate(s) of the compromised roots (emSign ECC Root CA - G3)", + ); + expect(infoMessage).toContain("(2 -> 1 certificates)"); + expect(infoMessage).toContain(`Node.js ${process.version}`); + expect(loggerStub.warn).not.toHaveBeenCalled(); + expect(loggerStub.error).not.toHaveBeenCalled(); + }); + + it("removes AAA Certificate Services, and every copy of a root listed more than once", () => { + const tlsStub = createTlsStub([ + EMSIGN_ECC_ROOT_G3_PEM, + AAA_CERTIFICATE_SERVICES_PEM, + ISRG_ROOT_X2_PEM, + EMSIGN_ECC_ROOT_G3_PEM, + ]); + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub }); + + expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]); + expect(loggerStub.info.mock.calls[0][0]).toContain( + "Removed 3 certificate(s) of the compromised roots (emSign ECC Root CA - G3, AAA Certificate Services)", + ); + }); + + it("sets a list without the roots unchanged and reports 0 removed", () => { + const tlsStub = createTlsStub([ISRG_ROOT_X2_PEM]); + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub }); + + expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]); + const infoMessage = loggerStub.info.mock.calls[0][0]; + expect(infoMessage).toContain("Removed 0 certificate(s) of the compromised roots from"); + expect(infoMessage).toContain("(1 -> 1 certificates)"); + }); + + it.each<[string, CaCertificateApi]>([ + ["tls.getCACertificates", { setDefaultCACertificates: vi.fn() }], + ["tls.setDefaultCACertificates", { getCACertificates: vi.fn(() => [EMSIGN_ECC_ROOT_G3_PEM]) }], + ])("without %s it warns and changes nothing", (_missingFunctionName, partialTlsStub) => { + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: partialTlsStub, logger: loggerStub }); + + for (const presentFunction of Object.values(partialTlsStub)) { + expect(presentFunction).not.toHaveBeenCalled(); + } + expect(loggerStub.warn).toHaveBeenCalledTimes(1); + expect(loggerStub.warn.mock.calls[0][0]).toContain( + `Node.js ${process.version} has no tls.getCACertificates`, + ); + expect(loggerStub.info).not.toHaveBeenCalled(); + expect(loggerStub.error).not.toHaveBeenCalled(); + }); + + it("logs an error from tls.getCACertificates with the error instead of throwing it", () => { + const readError = new Error("cannot read the CA list"); + const tlsStub = createTlsStub([]); + tlsStub.getCACertificates.mockImplementation(() => { + throw readError; + }); + const loggerStub = createLoggerStub(); + + expect(() => removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub })).not.toThrow(); + expect(tlsStub.setDefaultCACertificates).not.toHaveBeenCalled(); + expect(loggerStub.error).toHaveBeenCalledTimes(1); + const [errorMessage, loggedError] = loggerStub.error.mock.calls[0]; + expect(errorMessage).toContain( + `Failed to remove the compromised roots from Node's default CA list on Node.js ${process.version}`, + ); + expect(errorMessage).toContain("Routes keep serving."); + expect(loggedError).toBe(readError); + expect(loggerStub.info).not.toHaveBeenCalled(); + }); + + it("logs an unparseable certificate or an error from tls.setDefaultCACertificates instead of throwing", () => { + const unparseableTlsStub = createTlsStub([ + "-----BEGIN CERTIFICATE-----\nnot a certificate\n-----END CERTIFICATE-----\n", + ]); + const unparseableLoggerStub = createLoggerStub(); + expect(() => + removeCompromisedEmudhraRoots({ tls: unparseableTlsStub, logger: unparseableLoggerStub }), + ).not.toThrow(); + expect(unparseableTlsStub.setDefaultCACertificates).not.toHaveBeenCalled(); + expect(unparseableLoggerStub.error).toHaveBeenCalledTimes(1); + expect(unparseableLoggerStub.error.mock.calls[0][1]).toBeInstanceOf(Error); + + const writeError = new TypeError("bad certificate list"); + const rejectingTlsStub = createTlsStub([EMSIGN_ECC_ROOT_G3_PEM]); + rejectingTlsStub.setDefaultCACertificates.mockImplementation(() => { + throw writeError; + }); + const rejectingLoggerStub = createLoggerStub(); + expect(() => + removeCompromisedEmudhraRoots({ tls: rejectingTlsStub, logger: rejectingLoggerStub }), + ).not.toThrow(); + expect(rejectingLoggerStub.error.mock.calls[0][1]).toBe(writeError); + expect(rejectingLoggerStub.info).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/utils/compromisedCaRoots.ts b/frontend/src/utils/compromisedCaRoots.ts new file mode 100644 index 00000000..14f70588 --- /dev/null +++ b/frontend/src/utils/compromisedCaRoots.ts @@ -0,0 +1,109 @@ +// Server only: called from src/instrumentation.ts when a server instance +// starts, before the /api/proxy route forwards requests (with the user's +// Authorization header) to the Modal backend. +// +// Temporary. On 2026-10-20 it was disclosed that the private keys of eight +// eMudhra (emSign) intermediate CA certificates can be recovered +// (https://emudhra.we-have-a-problem.com). Six of them are TLS intermediates +// that chain to "emSign ECC Root CA - G3", and all eight are also cross-signed +// by Sectigo's "AAA Certificate Services". Vercel Functions ran Node.js +// 24.21.0 and 22.23.2 when last measured (2026-10-21), and a PolicyEngine +// preview function still ran 24.21.0 on 2026-10-26. The CA list compiled +// into both releases includes "emSign ECC Root CA - G3"; the one in Node.js +// 24.22.1 and 22.23.4 does not. This module removes both roots from +// Node's default CA list, which TLS connections opened after this call use +// (fetch included) unless they pass their own `ca` option. +// +// Delete this module, its test and src/instrumentation.ts once Vercel +// Functions run Node.js 24.22.1 / 22.23.4 or later. The info message below +// logs process.version each time a server instance starts, so the function +// logs show which version is running. +import { X509Certificate } from "node:crypto"; +import nodeTls from "node:tls"; + +const LOG_PREFIX = "[compromisedCaRoots]"; + +// Root certificates to remove, keyed by SHA-256 fingerprint in the format of +// X509Certificate#fingerprint256. Each fingerprint matches the CCADB "All +// Certificate Records" report and the PEM in Debian's ca-certificates +// 20230311. +export const COMPROMISED_ROOT_FINGERPRINTS: ReadonlyMap = new Map([ + // emSign ECC Root CA - G3 (eMudhra). Also matches the PEM in the CA list + // compiled into Node.js 24.21.0 and 22.23.2. + [ + "86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B", + "emSign ECC Root CA - G3", + ], + // AAA Certificate Services (Sectigo, formerly Comodo). Removed only as a + // defensive measure: the CA list compiled into Node.js 24.21.0 and 22.23.2 + // does not contain it, because Mozilla trusts it for email only and Node.js + // compiles in only the roots Mozilla trusts for websites. It can reach + // Node's default CA list through NODE_EXTRA_CA_CERTS, or through + // --use-system-ca (or NODE_USE_SYSTEM_CA=1) on an operating system whose CA + // store still contains it. + [ + "D7:A7:A0:FB:5D:7E:27:31:D7:71:E9:48:4E:BC:DE:F7:1D:5F:0C:3E:0A:29:48:78:2B:C8:3E:E0:EA:69:9E:F4", + "AAA Certificate Services", + ], +]); + +// The parts of node:tls this module uses. Both functions are optional because +// Node.js releases before 22.19 / 24.5 lack setDefaultCACertificates. +export type CaCertificateApi = { + getCACertificates?: (type: "default") => string[]; + setDefaultCACertificates?: (certificates: string[]) => void; +}; + +export type CaFilterLogger = Pick; + +// `tls` and `logger` are parameters so the unit test can pass stubs. +// +// An error is logged, not rethrown. Next.js rethrows an error from +// instrumentation's register() while it prepares the server, so a throw here +// would make every function-served route fail, the backend proxy included. +// The filter changes only which roots the functions' outbound HTTPS requests +// trust, so keeping the routes available is preferred; the error message says +// the roots may still be trusted. +export function removeCompromisedEmudhraRoots({ + tls = nodeTls, + logger = console, +}: { tls?: CaCertificateApi; logger?: CaFilterLogger } = {}): void { + if (typeof tls.getCACertificates !== "function" || typeof tls.setDefaultCACertificates !== "function") { + logger.warn( + `${LOG_PREFIX} Node.js ${process.version} has no tls.getCACertificates or ` + + "tls.setDefaultCACertificates; the compromised roots were not removed from Node's " + + "default CA list.", + ); + return; + } + try { + const defaultCertificates = tls.getCACertificates("default"); + // A root can appear more than once (an operating-system store added by + // --use-system-ca may list it several times), so count certificates and + // name each removed root once. + const removedRootNames = new Set(); + const keptCertificates = defaultCertificates.filter((pem) => { + const rootName = COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(pem).fingerprint256); + if (rootName === undefined) { + return true; + } + removedRootNames.add(rootName); + return false; + }); + tls.setDefaultCACertificates(keptCertificates); + const removedCount = defaultCertificates.length - keptCertificates.length; + const removedNamesText = removedCount > 0 ? ` (${[...removedRootNames].join(", ")})` : ""; + logger.info( + `${LOG_PREFIX} Removed ${removedCount} certificate(s) of the compromised roots` + + `${removedNamesText} from Node's default CA list (${defaultCertificates.length} -> ` + + `${keptCertificates.length} certificates) on Node.js ${process.version}. Delete this ` + + "module once Vercel Functions run Node.js 24.22.1 / 22.23.4 or later.", + ); + } catch (error) { + logger.error( + `${LOG_PREFIX} Failed to remove the compromised roots from Node's default CA list on ` + + `Node.js ${process.version}; the list may still trust them. Routes keep serving.`, + error, + ); + } +} diff --git a/modal_app.py b/modal_app.py index eba10da3..3d2ff24d 100644 --- a/modal_app.py +++ b/modal_app.py @@ -18,6 +18,24 @@ modal.Image.debian_slim(python_version="3.13") .apt_install("libpq-dev", "gcc") .pip_install_from_requirements("backend/requirements.txt") + # Refresh the CA lists the backend's outbound HTTPS and gRPC calls verify + # against, so they stop trusting the compromised eMudhra root "emSign ECC + # Root CA - G3" (https://emudhra.we-have-a-problem.com): + # - Debian's ca-certificates, read by clients that use the operating-system + # store, such as hf-xet for Hugging Face downloads. Modal's debian_slim + # base ships 20230311, which includes the root. + # - certifi, read by httpx and requests (the Anthropic, Supabase and Stripe + # clients). requirements.txt does not pin it, so the version is whatever + # was current when the step above was last built; 2026.10.19 is the first + # release without the root. A minimum, so a rebuild picks up later + # removals. + # - gRPC (Google Cloud Logging) reads the CA list bundled in grpcio, which + # no grpcio release has fixed; point it at the refreshed Debian bundle. + # These steps follow the requirements step so Modal reuses its cached + # build; changing an earlier step would re-resolve every unpinned package. + .apt_install("ca-certificates") + .pip_install("certifi>=2026.10.19") + .env({"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": "/etc/ssl/certs/ca-certificates.crt"}) .add_local_dir("backend", remote_path="/app/backend", copy=True) ) diff --git a/modal_eval_app.py b/modal_eval_app.py index c313519c..a167b293 100644 --- a/modal_eval_app.py +++ b/modal_eval_app.py @@ -34,6 +34,12 @@ image = ( modal.Image.debian_slim(python_version="3.13") .pip_install_from_requirements("backend/requirements-modal-eval.txt") + # httpx verifies the deployed backend's certificate against certifi, which + # the requirements file does not pin. 2026.10.19 is the first release that + # no longer trusts the compromised eMudhra root "emSign ECC Root CA - G3" + # (https://emudhra.we-have-a-problem.com). This step follows the + # requirements step so Modal reuses its cached build. + .pip_install("certifi>=2026.10.19") .add_local_dir("backend", remote_path="/app/backend", copy=True) .add_local_dir("evals", remote_path="/app/evals", copy=True) ) From 1c648d136a539ecc04300d197670c8eb2fad18b0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:20:18 +0400 Subject: [PATCH 2/2] Run the frontend CI jobs on Node.js 24 The frontend jobs in tests.yml and deploy.yml ran Node.js 20.19, which gets no release without the compromised eMudhra root "emSign ECC Root CA - G3". Under it, npm ci sends requests whose responses the lockfile's integrity hashes do not cover: the audit request (POST /-/npm/v1/security/advisories/bulk) and package-metadata requests for the packages the audit reports. Use Node.js 24, the version the Vercel project runs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 6 +++++- .github/workflows/tests.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3f1e6494..3c30b948 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -55,7 +55,11 @@ jobs: - name: Set up Node uses: actions/setup-node@v6 with: - node-version: "20.19" + # The Vercel project runs Node.js 24.x. Node.js 20 gets no release + # without the compromised eMudhra root "emSign ECC Root CA - G3", + # and npm ci's audit and package-metadata requests are not checked + # against the lockfile's integrity hashes. + node-version: "24" cache: npm cache-dependency-path: frontend/package-lock.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbeddbe8..b8727c40 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,7 +79,11 @@ jobs: - name: Set up Node uses: actions/setup-node@v6 with: - node-version: "20.19" + # The Vercel project runs Node.js 24.x. Node.js 20 gets no release + # without the compromised eMudhra root "emSign ECC Root CA - G3", + # and npm ci's audit and package-metadata requests are not checked + # against the lockfile's integrity hashes. + node-version: "24" cache: npm cache-dependency-path: frontend/package-lock.json