diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3f1e649..3c30b94 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -55,7 +55,11 @@ jobs: - name: Set up Node uses: actions/setup-node@v6 with: - node-version: "20.19" + # The Vercel project runs Node.js 24.x. Node.js 20 gets no release + # without the compromised eMudhra root "emSign ECC Root CA - G3", + # and npm ci's audit and package-metadata requests are not checked + # against the lockfile's integrity hashes. + node-version: "24" cache: npm cache-dependency-path: frontend/package-lock.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbeddbe..b8727c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,7 +79,11 @@ jobs: - name: Set up Node uses: actions/setup-node@v6 with: - node-version: "20.19" + # The Vercel project runs Node.js 24.x. Node.js 20 gets no release + # without the compromised eMudhra root "emSign ECC Root CA - G3", + # and npm ci's audit and package-metadata requests are not checked + # against the lockfile's integrity hashes. + node-version: "24" cache: npm cache-dependency-path: frontend/package-lock.json diff --git a/backend/tests/test_modal_app.py b/backend/tests/test_modal_app.py index 2e6d77f..bda680d 100644 --- a/backend/tests/test_modal_app.py +++ b/backend/tests/test_modal_app.py @@ -49,6 +49,9 @@ def pip_install_from_requirements(self, *args): def pip_install(self, *args): return self._step("pip_install", *args) + def env(self, *args): + return self._step("env", *args) + def add_local_dir(self, *args, **kwargs): return self._step("add_local_dir", *args, **kwargs) @@ -99,8 +102,18 @@ def test_modal_deployment_definition_imports_without_remote_calls(monkeypatch): "debian_slim", "apt_install", "pip_install_from_requirements", + "apt_install", + "pip_install", + "env", "add_local_dir", ] + # The CA-list refresh follows the requirements step, so Modal keeps + # that step's cached build (see the comment in modal_app.py). + assert modal_app.image.steps[3][1] == ("ca-certificates",) + assert modal_app.image.steps[4][1] == ("certifi>=2026.10.19",) + assert modal_app.image.steps[5][1] == ( + {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": "/etc/ssl/certs/ca-certificates.crt"}, + ) finally: sys.modules.pop("modal_app", None) @@ -140,6 +153,14 @@ def test_modal_eval_app_fans_out_twenty_cases_with_a_25_container_cap( "create_if_missing": True, "version": 2, } + assert [step[0] for step in modal_eval_app.image.steps] == [ + "debian_slim", + "pip_install_from_requirements", + "pip_install", + "add_local_dir", + "add_local_dir", + ] + assert modal_eval_app.image.steps[2][1] == ("certifi>=2026.10.19",) source = (REPO_ROOT / "modal_eval_app.py").read_text() assert "evaluate_case.spawn_map(" in source assert "concurrency=3" in source diff --git a/frontend/src/instrumentation.ts b/frontend/src/instrumentation.ts new file mode 100644 index 0000000..5bc77a4 --- /dev/null +++ b/frontend/src/instrumentation.ts @@ -0,0 +1,10 @@ +// Runs once per server instance, before it handles a request (Next.js 15 +// instrumentation). It removes the compromised eMudhra roots from Node's CA +// list before the /api/proxy route calls the Modal backend; see +// src/utils/compromisedCaRoots.ts for when to delete this file. +export async function register() { + if (process.env.NEXT_RUNTIME === "nodejs") { + const { removeCompromisedEmudhraRoots } = await import("./utils/compromisedCaRoots"); + removeCompromisedEmudhraRoots(); + } +} diff --git a/frontend/src/utils/compromisedCaRoots.test.ts b/frontend/src/utils/compromisedCaRoots.test.ts new file mode 100644 index 0000000..2a9e823 --- /dev/null +++ b/frontend/src/utils/compromisedCaRoots.test.ts @@ -0,0 +1,205 @@ +// @vitest-environment node +import { X509Certificate } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; + +import { + COMPROMISED_ROOT_FINGERPRINTS, + removeCompromisedEmudhraRoots, + type CaCertificateApi, +} from "./compromisedCaRoots"; + +// The two roots the module removes, as Debian's ca-certificates 20230311 +// ships them (emSign_ECC_Root_CA_-_G3.crt and Comodo_AAA_Services_root.crt). +const EMSIGN_ECC_ROOT_G3_PEM = `-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- +`; + +const AAA_CERTIFICATE_SERVICES_PEM = `-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- +`; + +// An unrelated root the module must keep, as certifi 2026.10.23 ships it. +const ISRG_ROOT_X2_PEM = `-----BEGIN CERTIFICATE----- +MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw +CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg +R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 +MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw +EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW ++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 +ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI +zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW +tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 +/q4AaOeMSQ+2b1tbFfLn +-----END CERTIFICATE----- +`; + +// A stand-in for node:tls whose default CA list starts as `defaultCertificates`. +function createTlsStub(defaultCertificates: string[]) { + return { + getCACertificates: vi.fn((_type: "default") => [...defaultCertificates]), + setDefaultCACertificates: vi.fn((_certificates: string[]) => {}), + }; +} + +function createLoggerStub() { + return { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; +} + +describe("removeCompromisedEmudhraRoots", () => { + it("keys each root by the SHA-256 fingerprint of its PEM", () => { + expect( + COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(EMSIGN_ECC_ROOT_G3_PEM).fingerprint256), + ).toBe("emSign ECC Root CA - G3"); + expect( + COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(AAA_CERTIFICATE_SERVICES_PEM).fingerprint256), + ).toBe("AAA Certificate Services"); + expect(COMPROMISED_ROOT_FINGERPRINTS.size).toBe(2); + }); + + it("removes emSign ECC Root CA - G3 and keeps an unrelated root", () => { + const tlsStub = createTlsStub([ISRG_ROOT_X2_PEM, EMSIGN_ECC_ROOT_G3_PEM]); + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub }); + + expect(tlsStub.getCACertificates).toHaveBeenCalledWith("default"); + expect(tlsStub.setDefaultCACertificates).toHaveBeenCalledTimes(1); + expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]); + expect(loggerStub.info).toHaveBeenCalledTimes(1); + const infoMessage = loggerStub.info.mock.calls[0][0]; + expect(infoMessage).toContain( + "Removed 1 certificate(s) of the compromised roots (emSign ECC Root CA - G3)", + ); + expect(infoMessage).toContain("(2 -> 1 certificates)"); + expect(infoMessage).toContain(`Node.js ${process.version}`); + expect(loggerStub.warn).not.toHaveBeenCalled(); + expect(loggerStub.error).not.toHaveBeenCalled(); + }); + + it("removes AAA Certificate Services, and every copy of a root listed more than once", () => { + const tlsStub = createTlsStub([ + EMSIGN_ECC_ROOT_G3_PEM, + AAA_CERTIFICATE_SERVICES_PEM, + ISRG_ROOT_X2_PEM, + EMSIGN_ECC_ROOT_G3_PEM, + ]); + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub }); + + expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]); + expect(loggerStub.info.mock.calls[0][0]).toContain( + "Removed 3 certificate(s) of the compromised roots (emSign ECC Root CA - G3, AAA Certificate Services)", + ); + }); + + it("sets a list without the roots unchanged and reports 0 removed", () => { + const tlsStub = createTlsStub([ISRG_ROOT_X2_PEM]); + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub }); + + expect(tlsStub.setDefaultCACertificates.mock.calls[0][0]).toEqual([ISRG_ROOT_X2_PEM]); + const infoMessage = loggerStub.info.mock.calls[0][0]; + expect(infoMessage).toContain("Removed 0 certificate(s) of the compromised roots from"); + expect(infoMessage).toContain("(1 -> 1 certificates)"); + }); + + it.each<[string, CaCertificateApi]>([ + ["tls.getCACertificates", { setDefaultCACertificates: vi.fn() }], + ["tls.setDefaultCACertificates", { getCACertificates: vi.fn(() => [EMSIGN_ECC_ROOT_G3_PEM]) }], + ])("without %s it warns and changes nothing", (_missingFunctionName, partialTlsStub) => { + const loggerStub = createLoggerStub(); + removeCompromisedEmudhraRoots({ tls: partialTlsStub, logger: loggerStub }); + + for (const presentFunction of Object.values(partialTlsStub)) { + expect(presentFunction).not.toHaveBeenCalled(); + } + expect(loggerStub.warn).toHaveBeenCalledTimes(1); + expect(loggerStub.warn.mock.calls[0][0]).toContain( + `Node.js ${process.version} has no tls.getCACertificates`, + ); + expect(loggerStub.info).not.toHaveBeenCalled(); + expect(loggerStub.error).not.toHaveBeenCalled(); + }); + + it("logs an error from tls.getCACertificates with the error instead of throwing it", () => { + const readError = new Error("cannot read the CA list"); + const tlsStub = createTlsStub([]); + tlsStub.getCACertificates.mockImplementation(() => { + throw readError; + }); + const loggerStub = createLoggerStub(); + + expect(() => removeCompromisedEmudhraRoots({ tls: tlsStub, logger: loggerStub })).not.toThrow(); + expect(tlsStub.setDefaultCACertificates).not.toHaveBeenCalled(); + expect(loggerStub.error).toHaveBeenCalledTimes(1); + const [errorMessage, loggedError] = loggerStub.error.mock.calls[0]; + expect(errorMessage).toContain( + `Failed to remove the compromised roots from Node's default CA list on Node.js ${process.version}`, + ); + expect(errorMessage).toContain("Routes keep serving."); + expect(loggedError).toBe(readError); + expect(loggerStub.info).not.toHaveBeenCalled(); + }); + + it("logs an unparseable certificate or an error from tls.setDefaultCACertificates instead of throwing", () => { + const unparseableTlsStub = createTlsStub([ + "-----BEGIN CERTIFICATE-----\nnot a certificate\n-----END CERTIFICATE-----\n", + ]); + const unparseableLoggerStub = createLoggerStub(); + expect(() => + removeCompromisedEmudhraRoots({ tls: unparseableTlsStub, logger: unparseableLoggerStub }), + ).not.toThrow(); + expect(unparseableTlsStub.setDefaultCACertificates).not.toHaveBeenCalled(); + expect(unparseableLoggerStub.error).toHaveBeenCalledTimes(1); + expect(unparseableLoggerStub.error.mock.calls[0][1]).toBeInstanceOf(Error); + + const writeError = new TypeError("bad certificate list"); + const rejectingTlsStub = createTlsStub([EMSIGN_ECC_ROOT_G3_PEM]); + rejectingTlsStub.setDefaultCACertificates.mockImplementation(() => { + throw writeError; + }); + const rejectingLoggerStub = createLoggerStub(); + expect(() => + removeCompromisedEmudhraRoots({ tls: rejectingTlsStub, logger: rejectingLoggerStub }), + ).not.toThrow(); + expect(rejectingLoggerStub.error.mock.calls[0][1]).toBe(writeError); + expect(rejectingLoggerStub.info).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/utils/compromisedCaRoots.ts b/frontend/src/utils/compromisedCaRoots.ts new file mode 100644 index 0000000..14f7058 --- /dev/null +++ b/frontend/src/utils/compromisedCaRoots.ts @@ -0,0 +1,109 @@ +// Server only: called from src/instrumentation.ts when a server instance +// starts, before the /api/proxy route forwards requests (with the user's +// Authorization header) to the Modal backend. +// +// Temporary. On 2026-10-20 it was disclosed that the private keys of eight +// eMudhra (emSign) intermediate CA certificates can be recovered +// (https://emudhra.we-have-a-problem.com). Six of them are TLS intermediates +// that chain to "emSign ECC Root CA - G3", and all eight are also cross-signed +// by Sectigo's "AAA Certificate Services". Vercel Functions ran Node.js +// 24.21.0 and 22.23.2 when last measured (2026-10-21), and a PolicyEngine +// preview function still ran 24.21.0 on 2026-10-26. The CA list compiled +// into both releases includes "emSign ECC Root CA - G3"; the one in Node.js +// 24.22.1 and 22.23.4 does not. This module removes both roots from +// Node's default CA list, which TLS connections opened after this call use +// (fetch included) unless they pass their own `ca` option. +// +// Delete this module, its test and src/instrumentation.ts once Vercel +// Functions run Node.js 24.22.1 / 22.23.4 or later. The info message below +// logs process.version each time a server instance starts, so the function +// logs show which version is running. +import { X509Certificate } from "node:crypto"; +import nodeTls from "node:tls"; + +const LOG_PREFIX = "[compromisedCaRoots]"; + +// Root certificates to remove, keyed by SHA-256 fingerprint in the format of +// X509Certificate#fingerprint256. Each fingerprint matches the CCADB "All +// Certificate Records" report and the PEM in Debian's ca-certificates +// 20230311. +export const COMPROMISED_ROOT_FINGERPRINTS: ReadonlyMap = new Map([ + // emSign ECC Root CA - G3 (eMudhra). Also matches the PEM in the CA list + // compiled into Node.js 24.21.0 and 22.23.2. + [ + "86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B", + "emSign ECC Root CA - G3", + ], + // AAA Certificate Services (Sectigo, formerly Comodo). Removed only as a + // defensive measure: the CA list compiled into Node.js 24.21.0 and 22.23.2 + // does not contain it, because Mozilla trusts it for email only and Node.js + // compiles in only the roots Mozilla trusts for websites. It can reach + // Node's default CA list through NODE_EXTRA_CA_CERTS, or through + // --use-system-ca (or NODE_USE_SYSTEM_CA=1) on an operating system whose CA + // store still contains it. + [ + "D7:A7:A0:FB:5D:7E:27:31:D7:71:E9:48:4E:BC:DE:F7:1D:5F:0C:3E:0A:29:48:78:2B:C8:3E:E0:EA:69:9E:F4", + "AAA Certificate Services", + ], +]); + +// The parts of node:tls this module uses. Both functions are optional because +// Node.js releases before 22.19 / 24.5 lack setDefaultCACertificates. +export type CaCertificateApi = { + getCACertificates?: (type: "default") => string[]; + setDefaultCACertificates?: (certificates: string[]) => void; +}; + +export type CaFilterLogger = Pick; + +// `tls` and `logger` are parameters so the unit test can pass stubs. +// +// An error is logged, not rethrown. Next.js rethrows an error from +// instrumentation's register() while it prepares the server, so a throw here +// would make every function-served route fail, the backend proxy included. +// The filter changes only which roots the functions' outbound HTTPS requests +// trust, so keeping the routes available is preferred; the error message says +// the roots may still be trusted. +export function removeCompromisedEmudhraRoots({ + tls = nodeTls, + logger = console, +}: { tls?: CaCertificateApi; logger?: CaFilterLogger } = {}): void { + if (typeof tls.getCACertificates !== "function" || typeof tls.setDefaultCACertificates !== "function") { + logger.warn( + `${LOG_PREFIX} Node.js ${process.version} has no tls.getCACertificates or ` + + "tls.setDefaultCACertificates; the compromised roots were not removed from Node's " + + "default CA list.", + ); + return; + } + try { + const defaultCertificates = tls.getCACertificates("default"); + // A root can appear more than once (an operating-system store added by + // --use-system-ca may list it several times), so count certificates and + // name each removed root once. + const removedRootNames = new Set(); + const keptCertificates = defaultCertificates.filter((pem) => { + const rootName = COMPROMISED_ROOT_FINGERPRINTS.get(new X509Certificate(pem).fingerprint256); + if (rootName === undefined) { + return true; + } + removedRootNames.add(rootName); + return false; + }); + tls.setDefaultCACertificates(keptCertificates); + const removedCount = defaultCertificates.length - keptCertificates.length; + const removedNamesText = removedCount > 0 ? ` (${[...removedRootNames].join(", ")})` : ""; + logger.info( + `${LOG_PREFIX} Removed ${removedCount} certificate(s) of the compromised roots` + + `${removedNamesText} from Node's default CA list (${defaultCertificates.length} -> ` + + `${keptCertificates.length} certificates) on Node.js ${process.version}. Delete this ` + + "module once Vercel Functions run Node.js 24.22.1 / 22.23.4 or later.", + ); + } catch (error) { + logger.error( + `${LOG_PREFIX} Failed to remove the compromised roots from Node's default CA list on ` + + `Node.js ${process.version}; the list may still trust them. Routes keep serving.`, + error, + ); + } +} diff --git a/modal_app.py b/modal_app.py index eba10da..3d2ff24 100644 --- a/modal_app.py +++ b/modal_app.py @@ -18,6 +18,24 @@ modal.Image.debian_slim(python_version="3.13") .apt_install("libpq-dev", "gcc") .pip_install_from_requirements("backend/requirements.txt") + # Refresh the CA lists the backend's outbound HTTPS and gRPC calls verify + # against, so they stop trusting the compromised eMudhra root "emSign ECC + # Root CA - G3" (https://emudhra.we-have-a-problem.com): + # - Debian's ca-certificates, read by clients that use the operating-system + # store, such as hf-xet for Hugging Face downloads. Modal's debian_slim + # base ships 20230311, which includes the root. + # - certifi, read by httpx and requests (the Anthropic, Supabase and Stripe + # clients). requirements.txt does not pin it, so the version is whatever + # was current when the step above was last built; 2026.10.19 is the first + # release without the root. A minimum, so a rebuild picks up later + # removals. + # - gRPC (Google Cloud Logging) reads the CA list bundled in grpcio, which + # no grpcio release has fixed; point it at the refreshed Debian bundle. + # These steps follow the requirements step so Modal reuses its cached + # build; changing an earlier step would re-resolve every unpinned package. + .apt_install("ca-certificates") + .pip_install("certifi>=2026.10.19") + .env({"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": "/etc/ssl/certs/ca-certificates.crt"}) .add_local_dir("backend", remote_path="/app/backend", copy=True) ) diff --git a/modal_eval_app.py b/modal_eval_app.py index c313519..a167b29 100644 --- a/modal_eval_app.py +++ b/modal_eval_app.py @@ -34,6 +34,12 @@ image = ( modal.Image.debian_slim(python_version="3.13") .pip_install_from_requirements("backend/requirements-modal-eval.txt") + # httpx verifies the deployed backend's certificate against certifi, which + # the requirements file does not pin. 2026.10.19 is the first release that + # no longer trusts the compromised eMudhra root "emSign ECC Root CA - G3" + # (https://emudhra.we-have-a-problem.com). This step follows the + # requirements step so Modal reuses its cached build. + .pip_install("certifi>=2026.10.19") .add_local_dir("backend", remote_path="/app/backend", copy=True) .add_local_dir("evals", remote_path="/app/evals", copy=True) )