fix(dns): stop the resolver putting a registry lookup in front of the whole web

lib/dns/policy.ts

@@ -38,6 +38,43 @@ export const NEVER_MOSHPIT = new Set([
   "localhost", "local", "onion", "test", "invalid", "example", "home", "internal", "lan",
 ]);
 
+/**
+ * Two-label endings that are registry boundaries in the legacy root, not names.
+ *
+ * The Moshpit namespace is one level deep, so the *last two labels* of a query
+ * are the name — which is right for `www.scrambled.eggs` and wrong for
+ * `www.bbc.co.uk`, where the last two labels are `co.uk`. That read the BBC as
+ * a name in this namespace: a registry lookup on every UK page load, and in
+ * `moshpit` mode, whoever registered `co.uk` would have intercepted every site
+ * under it.
+ *
+ * A bundled list goes stale — the same objection `roots.ts` raises — but this
+ * one ages far better than a gTLD list: ccTLD second levels change on the order
+ * of years, and being wrong costs one needless lookup rather than a wrong
+ * answer. It is not the full Public Suffix List, just the endings a browser is
+ * likely to meet.
+ */
+export const PUBLIC_SUFFIXES = new Set([
+  "co.uk", "org.uk", "ac.uk", "gov.uk", "me.uk", "net.uk", "sch.uk", "ltd.uk", "plc.uk",
+  "com.au", "net.au", "org.au", "edu.au", "gov.au", "id.au",
+  "co.jp", "or.jp", "ne.jp", "ac.jp", "go.jp", "lg.jp",
+  "co.nz", "net.nz", "org.nz", "govt.nz", "ac.nz",
+  "co.za", "org.za", "web.za", "gov.za", "ac.za",
+  "com.br", "net.br", "org.br", "gov.br", "edu.br",
+  "com.cn", "net.cn", "org.cn", "gov.cn", "edu.cn", "ac.cn",
+  "co.in", "net.in", "org.in", "gov.in", "ac.in", "edu.in",
+  "co.kr", "or.kr", "ne.kr", "go.kr", "re.kr",
+  "com.mx", "org.mx", "gob.mx", "edu.mx",
+  "com.tr", "net.tr", "org.tr", "gov.tr", "edu.tr",
+  "com.tw", "org.tw", "gov.tw", "edu.tw",
+  "com.sg", "net.sg", "org.sg", "gov.sg", "edu.sg",
+  "com.hk", "org.hk", "gov.hk", "edu.hk", "idv.hk",
+  "com.ar", "com.co", "com.pe", "com.uy", "com.ec", "com.ve",
+  "com.ua", "com.pl", "com.ru", "com.es", "com.pt", "com.gr", "com.cy",
+  "com.vn", "com.my", "com.ph", "com.pk", "com.bd", "com.np",
+  "com.eg", "com.sa", "com.ng", "com.gh", "com.kw", "com.qa",
+]);
+
 const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
 
 /**
@@ -75,6 +112,8 @@ export function moshpitCandidate(qname: string): { name: string; label: string;
   if (parts.every((part) => /^\d+$/.test(part))) return null;
   if (tld.length < 2) return null;
   if (!LABEL.test(tld) || !LABEL.test(label)) return null;
+  // `co.uk` is where names start, not a name. See PUBLIC_SUFFIXES.
+  if (PUBLIC_SUFFIXES.has(`${label}.${tld}`)) return null;
   // Underscore-prefixed service labels (`_dmarc`, `_acme-challenge`) are legal
   // in DNS but never a registry label, so they can only be a prefix.
   const prefix = parts.slice(0, parts.length - 2).join(".");
@@ -149,11 +188,30 @@ export function planQuery(opts: {
 /**
  * Did the upstream answer actually resolve the name?
  *
- * NXDOMAIN is the obvious no. NOERROR with an empty answer section (NODATA)
- * is the subtle one: the name exists in clearnet but has no address, e.g. a
- * parked domain with only an MX. Treating that as "clearnet answered" would
- * strand a Moshpit name behind a clearnet placeholder, so an address query
- * with no addresses counts as no answer.
+ * NXDOMAIN is the obvious no: clearnet has never heard of the name, so the
+ * registry gets a turn. NOERROR with records is the obvious yes.
+ *
+ * NOERROR with an empty answer section (NODATA) is the subtle one, and reading
+ * it as "clearnet has nothing" — which this used to do, for every query type —
+ * is what made the ordinary web slow.
+ *
+ * NODATA is a *positive* statement about the name: the zone exists and was
+ * asked, it simply holds no record of this type. It is also the common case,
+ * not the rare one. A browser asks A, AAAA and HTTPS (type 65) for every
+ * hostname it touches, and the vast majority of real domains have no AAAA and
+ * no HTTPS record — so two queries in three came back NODATA, were read as
+ * "clearnet came up empty", and paid for a registry round trip plus a root
+ * probe before the browser got its answer. Worse than slow: where the registry
+ * happened to hold that name, a legitimate domain's AAAA was answered with the
+ * *gateway's* address, and a dual-stack client went to the pit instead of the
+ * site it asked for.
+ *
+ * So NODATA now counts as an answer, with one exception kept deliberately: an
+ * `A` query with no addresses. That is the case the original note was about — a
+ * name that exists in clearnet with only an MX behind it, where backfilling
+ * from the registry is the useful thing to do — and it is genuinely rare, so it
+ * costs a lookup almost nowhere. Every other type is left to clearnet, which is
+ * the one that actually knows.
  */
 export function clearnetAnswered(response: {
   flags?: { rcode?: number };
@@ -163,5 +221,10 @@ export function clearnetAnswered(response: {
   const rcode = response?.flags?.rcode ?? RCODE.NOERROR;
   if (rcode === RCODE.NXDOMAIN) return false;
   if (rcode !== RCODE.NOERROR) return true; // SERVFAIL and friends: not ours to override
-  return (response?.answers?.length ?? 0) > 0;
+  if ((response?.answers?.length ?? 0) > 0) return true;
+
+  // NODATA. Only an address query with no address leaves room for a backfill;
+  // an unknown question type is left alone, because inventing an answer for a
+  // type we did not understand is how a resolver breaks things it never saw.
+  return response?.questions?.[0]?.type !== TYPE.A;
 }

lib/dns/ratelimit.ts

@@ -16,6 +16,28 @@ export type RateLimiter = {
   size(): number;
 };
 
+/**
+ * The local machine, which is never rate limited.
+ *
+ * The limit above prices an attack that needs a forged source address, and
+ * loopback cannot be forged from off the box: the kernel drops a 127/8 source
+ * arriving on a real interface. So there is nothing to price here.
+ *
+ * There is a great deal to lose, though, the moment a machine points its own
+ * resolver at this (`DNS=127.0.0.1:5354`). Every query on that machine then
+ * arrives from one address and shares a single bucket sized for one client, so
+ * the sustained rate becomes the whole machine's DNS budget. A page load bursts
+ * well past it, and over-budget queries are dropped rather than refused —
+ * correct against a spoofing victim, and the worst possible answer locally,
+ * where it means the stub waits out a full timeout, retries, and the page loads
+ * with subresources that never resolved.
+ */
+const LOOPBACK = /^(?:127\.\d{1,3}\.\d{1,3}\.\d{1,3}|::1|::ffff:127\.\d{1,3}\.\d{1,3}\.\d{1,3})$/;
+
+export function isLoopback(remote: string | undefined): boolean {
+  return LOOPBACK.test(String(remote ?? "").trim().toLowerCase());
+}
+
 export function createRateLimiter(options: {
   /** Sustained queries per second, per client. */
   qps?: number;

lib/dns/server.ts

@@ -12,7 +12,7 @@ import { moshpitAnswer, type GatewayAddresses } from "./answers";
 import type { GatewayResolver } from "./gateway";
 import { clearnetAnswered, planQuery, type ResolveMode } from "./policy";
 import type { RootProbe } from "./roots";
-import type { RateLimiter } from "./ratelimit";
+import { isLoopback, type RateLimiter } from "./ratelimit";
 import type { RegistryClient } from "./registry";
 import type { Forwarder } from "./upstream";
 import {
@@ -404,6 +404,8 @@ export function createDnsServer(options: DnsServerOptions): DnsServer {
 
   function allowed(remote: string | undefined): boolean {
     if (!options.rateLimiter || !remote) return true;
+    // The machine's own queries are never throttled — see isLoopback.
+    if (isLoopback(remote)) return true;
     if (options.rateLimiter.allow(remote)) return true;
     stats.dropped++;
     return false;

lib/dns/wire.ts

@@ -20,6 +20,13 @@ export const TYPE = {
   TXT: 16,
   AAAA: 28,
   OPT: 41,
+  // Not encoded or decoded as records — the resolver only ever relays these —
+  // but named because policy has to reason about them. A browser asks HTTPS
+  // (RFC 9460) for every hostname it navigates to, alongside A and AAAA, and
+  // most domains answer it with nothing. Which of those empty answers mean
+  // "clearnet has no name here" is the whole of `clearnetAnswered`.
+  SVCB: 64,
+  HTTPS: 65,
   ANY: 255,
 } as const;
 

tests/dns-policy.test.mjs

@@ -94,12 +94,44 @@ test("without recursion we answer only what we are authoritative for", () => {
   assert.equal(planQuery({ question: question("example.com"), rd: false }).action, "refuse");
 });
 
-test("NODATA from clearnet counts as no answer, so the registry still gets a turn", () => {
+test("clearnet owns a name it has heard of, whatever it was asked for", () => {
+  const nodata = (type) => ({ flags: { rcode: RCODE.NOERROR }, answers: [], questions: [question("thing.dev", type)] });
+
   assert.equal(clearnetAnswered({ flags: { rcode: RCODE.NXDOMAIN }, answers: [] }), false);
-  assert.equal(clearnetAnswered({ flags: { rcode: RCODE.NOERROR }, answers: [] }), false);
   assert.equal(clearnetAnswered({ flags: { rcode: RCODE.NOERROR }, answers: [{ type: TYPE.A }] }), true);
   // A broken upstream is not permission to substitute our own namespace.
   assert.equal(clearnetAnswered({ flags: { rcode: RCODE.SERVFAIL }, answers: [] }), true);
+
+  // NODATA says the name exists and has no record of THIS type. A browser asks
+  // A, AAAA and HTTPS for every hostname and most real domains answer the last
+  // two with nothing — so reading those as "clearnet came up empty" put a
+  // registry round trip in front of most of the web, and handed the gateway's
+  // address to anyone whose domain the registry happened to hold.
+  assert.equal(clearnetAnswered(nodata(TYPE.AAAA)), true);
+  assert.equal(clearnetAnswered(nodata(TYPE.HTTPS)), true);
+  assert.equal(clearnetAnswered(nodata(TYPE.MX)), true);
+
+  // The one case kept: an address query with no address, e.g. a clearnet name
+  // parked behind an MX. Rare, so it costs a lookup almost nowhere.
+  assert.equal(clearnetAnswered(nodata(TYPE.A)), false);
+
+  // An answer we cannot attribute to a question is left to clearnet.
+  assert.equal(clearnetAnswered({ flags: { rcode: RCODE.NOERROR }, answers: [] }), true);
+});
+
+test("a country-code second level is where names start, not a name", () => {
+  // The last two labels of `www.bbc.co.uk` are `co.uk`. Reading that as a
+  // Moshpit name meant a registry lookup on every UK page load, and in moshpit
+  // mode it would have handed every site under `.co.uk` to whoever held it.
+  for (const name of ["bbc.co.uk", "www.bbc.co.uk", "abc.net.au", "asahi.co.jp", "gov.uk.com.br"]) {
+    const candidate = moshpitCandidate(name);
+    assert.notEqual(candidate?.name, "co.uk", `${name} must not resolve to the co.uk suffix`);
+  }
+  assert.equal(moshpitCandidate("bbc.co.uk"), null);
+  assert.equal(moshpitCandidate("www.bbc.co.uk"), null);
+  assert.equal(moshpitCandidate("abc.net.au"), null);
+  // A real Moshpit name that merely ends in a ccTLD-looking label still works.
+  assert.equal(moshpitCandidate("scrambled.eggs")?.name, "scrambled.eggs");
 });
 
 test("a registered name answers with the gateway's addresses", () => {

tests/dns-server.test.mjs

@@ -8,7 +8,7 @@ import net from "node:net";
 import test from "node:test";
 
 import { createGatewayResolver } from "../lib/dns/gateway.ts";
-import { createRateLimiter } from "../lib/dns/ratelimit.ts";
+import { createRateLimiter, isLoopback } from "../lib/dns/ratelimit.ts";
 import { createRegistryClient } from "../lib/dns/registry.ts";
 import { createDnsServer } from "../lib/dns/server.ts";
 import { createForwarder, parseUpstreams } from "../lib/dns/upstream.ts";
@@ -200,6 +200,67 @@ test("a client over its rate limit is dropped rather than answered", async () =>
   assert.equal(limiter.allow("192.0.2.2"), true, "a different client has its own budget");
 });
 
+test("the machine's own queries are never rate limited", () => {
+  // Point a box's resolver at this and every query on it arrives from one
+  // address. Under a per-client limit that made the whole machine share one
+  // client's budget, and the excess was dropped in silence — a page load's
+  // worth of subresources that simply never resolved.
+  for (const local of ["127.0.0.1", "127.0.0.53", "::1", "::ffff:127.0.0.1"]) {
+    assert.equal(isLoopback(local), true, `${local} is the local machine`);
+  }
+  for (const remote of ["192.0.2.1", "203.0.113.7", "2606:4700::1111", "", undefined]) {
+    assert.equal(isLoopback(remote), false, `${remote} is not loopback and still pays the limit`);
+  }
+});
+
+test("a clearnet name with no AAAA is left to clearnet, not answered from the registry", async () => {
+  // The regression this exists for: a browser asks A, AAAA and HTTPS for every
+  // hostname, and most real domains answer the last two with NODATA. Reading
+  // that as "clearnet came up empty" put a registry round trip in front of most
+  // of the web — and where the registry held the name, it answered a real
+  // domain's AAAA with the *gateway*, sending dual-stack clients to the pit.
+  const forwarder = createForwarder({
+    ask: async (_upstream, payload) => {
+      const q = decodeMessage(payload);
+      const question = q.questions[0];
+      const isA = question.type === TYPE.A;
+      return encodeMessage({
+        id: q.id,
+        flags: { qr: true, rd: true, ra: true, rcode: RCODE.NOERROR },
+        questions: q.questions,
+        // An ordinary dual-stack-less domain: an address, and nothing else.
+        answers: isA
+          ? [{ name: question.name, type: TYPE.A, class: CLASS.IN, ttl: 300, address: CLEARNET_V4 }]
+          : [],
+      });
+    },
+  });
+  const registry = stubRegistry({ "profullstack.ai": {} });
+  const dns = createDnsServer({
+    registry,
+    forwarder,
+    gateway: createGatewayResolver({ host: "pit.moshcode.sh", forwarder, ipv4: [GATEWAY_V4], ipv6: ["2606:4700::1111"] }),
+    mode: "clearnet",
+    ttl: 60,
+    port: 0,
+    address: "127.0.0.1",
+  });
+
+  const v6 = decodeMessage(await dns.handle(query("profullstack.ai", TYPE.AAAA)));
+  assert.deepEqual(addresses(v6, TYPE.AAAA), [], "the gateway's address must not stand in for a real domain's AAAA");
+  assert.equal(v6.flags.aa, false, "clearnet's own NODATA is relayed, not replaced with an authoritative one");
+
+  const https = decodeMessage(await dns.handle(query("profullstack.ai", TYPE.HTTPS)));
+  assert.equal(https.answers.length, 0);
+
+  assert.equal(dns.stats().moshpit, 0, "no Moshpit answer was synthesized");
+  assert.equal(registry.stats().misses, 0, "and the registry was never asked");
+
+  // The A query still resolves through clearnet, as it always did.
+  assert.deepEqual(addresses(decodeMessage(await dns.handle(query("profullstack.ai")))), [CLEARNET_V4]);
+  await dns.close();
+});
+
 test("EDNS is echoed, so clients keep using it", async () => {
   const dns = harness({ names: { "scrambled.eggs": {} } });
   const withEdns = encodeMessage({