feat: stop counting bots and headless crawlers in Umami

.github/workflows/ci.yml

@@ -86,8 +86,9 @@ jobs:
     # Builds with the production address, serves dist/ the way Pages does (scripts/serve.ts) and
     # runs Lighthouse three times on each URL in lighthouserc.json, as a phone. A median score
     # under 90 for performance, accessibility, best practices or SEO on any URL fails the job.
-    # Umami's script loads here as for any reader, but it counts only on docs.sluiceway.dev, so
-    # these runs on localhost send nothing.
+    # Umami's script does not load here: Lighthouse names itself in its user agent, so the docs
+    # treat it as automated (src/lib/automated.ts). It would count only on docs.sluiceway.dev
+    # anyway.
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:

README.md

@@ -71,7 +71,7 @@ The reports are in `.lighthouseci/`. Set `CHROME_PATH` if Lighthouse cannot find
 
 ## Analytics
 
-The Umami website id is `DEFAULT_UMAMI_WEBSITE_ID` in `src/lib/analytics-build.ts`, and `PUBLIC_UMAMI_WEBSITE_ID` overrides it at build time. An empty `PUBLIC_UMAMI_WEBSITE_ID` fails the build. To build without analytics, in a fork or locally, set `ANALYTICS_OFF=1` for the build and for `bun test`. `test/analytics-built.test.ts` follows each sampled page's scripts and their imports in `dist/` and fails when the script URL or the id is missing, so analytics cannot go quiet unnoticed.
+Umami is not loaded for a program rather than a person: a browser driven by automation, a user agent that names a bot, crawler, spider, headless browser or link preview, or a screen of exactly 800x600 or 1024x1024 (`src/lib/automated.ts`, the same rule as in sluiceway/landing and sluiceway/app). The Umami website id is `DEFAULT_UMAMI_WEBSITE_ID` in `src/lib/analytics-build.ts`, and `PUBLIC_UMAMI_WEBSITE_ID` overrides it at build time. An empty `PUBLIC_UMAMI_WEBSITE_ID` fails the build. To build without analytics, in a fork or locally, set `ANALYTICS_OFF=1` for the build and for `bun test`. `test/analytics-built.test.ts` follows each sampled page's scripts and their imports in `dist/` and fails when the script URL or the id is missing, so analytics cannot go quiet unnoticed.
 
 ## Updating to a new release of Sluiceway
 

src/components/Analytics.astro

@@ -1,6 +1,6 @@
 ---
 // Analytics: starts Umami on the page and hooks up the named events, for every reader but one
-// whose browser sends Do Not Track. With analytics off, `init()` adds nothing and every event
+// whose browser sends Do Not Track or who is a program (src/lib/automated.ts). With analytics off, `init()` adds nothing and every event
 // is dropped.
 ---
 

src/components/Feedback.astro

@@ -50,7 +50,9 @@ import PennyMark from "./PennyMark.astro";
             ? "Thanks."
             : analytics?.blocked()
               ? "Thanks. Your browser sends Do Not Track, so this was not sent."
-              : "Thanks. Analytics is off, so this was not sent.";
+              : analytics?.automated()
+                ? "Thanks. This browser looks automated, so this was not sent."
+                : "Thanks. Analytics is off, so this was not sent.";
           // The pressed button is gone, so keyboard focus moves to the reply.
           if (hadFocus) reply.focus();
         }

src/content/docs/privacy.mdx

@@ -60,6 +60,13 @@ your browser clears when you close the tab.
 
 If your browser sends Do Not Track, the Umami script is not loaded and nothing is counted.
 
+## Automated visitors
+
+Visits from programs are not counted. The Umami script is not loaded and nothing is sent when
+your browser says it is driven by automation, when its user agent names a bot, a crawler, a
+spider, a headless browser or a link preview service, or when its screen is
+exactly 800 by 600 or 1024 by 1024 pixels, the sizes headless crawlers report.
+
 ## How to object
 
 Email [robbe.verhelst@gmail.com](mailto:robbe.verhelst@gmail.com) to object, or to ask for what

src/lib/analytics.ts

@@ -1,9 +1,11 @@
 // Analytics with self-hosted Umami, for every reader. Umami sets no cookie and these docs
 // store nothing in the browser, so there is nothing to ask. Do Not Track blocks everything: the
-// script is never added and no event is sent. The privacy page says the same in words, so a
+// script is never added and no event is sent. So does a visit from a program rather than a
+// person, by the rule in ./automated.ts. The privacy page says the same in words, so a
 // change here is a change there (src/content/docs/privacy.mdx).
 
 import { UMAMI_SCRIPT_URL } from "./analytics-build";
+import { isAutomated } from "./automated";
 
 /**
  * The website id, set by astro.config.ts from `umamiWebsiteId()`: the docs' own id, an
@@ -37,7 +39,8 @@ declare global {
 /** What analytics needs from the browser, so the tests can hand it a fake one. */
 export interface AnalyticsEnv {
   storage: Pick<Storage, "removeItem"> | undefined;
-  navigator: { doNotTrack?: string | null };
+  navigator: { doNotTrack?: string | null; webdriver?: boolean; userAgent?: string };
+  screen: { width: number; height: number };
   /** Add a deferred `<script>` with these attributes, and call `onload` once it has run. */
   addScript(attributes: Record<string, string>, onload: () => void): void;
   umami(): Umami | undefined;
@@ -54,11 +57,13 @@ export interface Analytics {
   readonly enabled: boolean;
   /** Do Not Track is on. Nothing loads and nothing is sent. */
   blocked(): boolean;
+  /** A bot, a crawler, a headless browser or a link preview. Nothing loads and nothing is sent. */
+  automated(): boolean;
   /** Analytics may run on this page. */
   allowed(): boolean;
-  /** Add the script, unless analytics is off or blocked. Call once per page. */
+  /** Add the script, unless analytics is off, blocked or automated. Call once per page. */
   init(): void;
-  /** Send a named event. Off or blocked, it sends nothing and returns false. */
+  /** Send a named event. Off, blocked or automated, it sends nothing and returns false. */
   track(event: string, data?: EventData): boolean;
 }
 
@@ -69,7 +74,13 @@ export function createAnalytics(config: AnalyticsConfig, env: AnalyticsEnv): Ana
   const queue: [string, EventData | undefined][] = [];
 
   const blocked = () => env.navigator.doNotTrack === "1";
-  const allowed = () => enabled && !blocked();
+  const automated = () =>
+    isAutomated({
+      webdriver: env.navigator.webdriver === true,
+      userAgent: env.navigator.userAgent ?? "",
+      screen: env.screen,
+    });
+  const allowed = () => enabled && !blocked() && !automated();
 
   const load = () => {
     if (state !== "none" || !allowed()) return;
@@ -93,6 +104,7 @@ export function createAnalytics(config: AnalyticsConfig, env: AnalyticsEnv): Ana
   return {
     enabled,
     blocked,
+    automated,
     allowed,
     init() {
       try {
@@ -122,6 +134,7 @@ function browserEnv(): AnalyticsEnv {
   return {
     storage,
     navigator: navigator as AnalyticsEnv["navigator"],
+    screen: { width: window.screen.width, height: window.screen.height },
     addScript(attributes, onload) {
       const script = document.createElement("script");
       script.defer = true;

src/lib/automated.ts

@@ -0,0 +1,38 @@
+// Whether this visit comes from a program rather than a person. Nearly every
+// visit Umami counted besides the owner's was a headless crawler: a stock
+// Chrome, Edge or Safari user agent on a screen of exactly 800x600 (once
+// 1024x1024), no clicks, often the same page twice a day apart. Those are not
+// counted. The same rule stands in sluiceway/landing and sluiceway/app; change
+// the three together (src/content/docs/privacy.mdx says it in words).
+//
+// A visit is automated when any of these holds:
+// - the browser says it is driven by automation (navigator.webdriver);
+// - the user agent names a bot, a crawler, a spider, a headless browser or a
+//   link preview service;
+// - the screen is exactly 800x600 or 1024x1024, the sizes headless crawlers
+//   report and no person's screen was seen at.
+
+export type Visitor = {
+  webdriver: boolean;
+  userAgent: string;
+  screen: { width: number; height: number };
+};
+
+// Bots, crawlers and spiders, headless browsers and the tools that drive
+// them, and the services that fetch a page to show a preview of a link.
+// "Cubot" is a phone maker, not a bot.
+export const AUTOMATED_USER_AGENT =
+  /(?<!cu)bot(?![a-z])|crawl|spider|slurp|headless|phantomjs|puppeteer|playwright|selenium|webdriver|lighthouse|preview|facebookexternalhit|facebookcatalog|embedly|iframely|whatsapp|skypeuripreview|google-pagerenderer|googleother|google-inspectiontool/i;
+
+export const AUTOMATED_SCREENS: readonly (readonly [number, number])[] = [
+  [800, 600],
+  [1024, 1024],
+];
+
+export function isAutomated({ webdriver, userAgent, screen }: Visitor): boolean {
+  if (webdriver) return true;
+  if (AUTOMATED_USER_AGENT.test(userAgent)) return true;
+  return AUTOMATED_SCREENS.some(
+    ([width, height]) => screen.width === width && screen.height === height,
+  );
+}

test/analytics-built.test.ts

@@ -129,6 +129,10 @@ describe(ON ? "analytics on" : "analytics off (ANALYTICS_OFF=1)", () => {
         expect(allScripts(graph)).toContain("doNotTrack");
       });
 
+      test(`${page} checks for an automated browser before loading`, () => {
+        expect(allScripts(graph)).toContain("webdriver");
+      });
+
       test(`${page} carries the footer note`, () => {
         expect(graph.html).toContain("sw-analytics-note");
       });

test/analytics.test.ts

@@ -19,7 +19,10 @@ const CONFIG = {
  * A fake browser: records every script added, every event Umami receives, and every write to
  * storage. It starts with an old consent record in storage, as a returning reader's has.
  */
-function fakeBrowser(navigator: AnalyticsEnv["navigator"] = {}) {
+function fakeBrowser(
+  navigator: AnalyticsEnv["navigator"] = {},
+  screen: AnalyticsEnv["screen"] = { width: 1920, height: 1080 },
+) {
   const store = new Map<string, string>([[OLD_CONSENT_KEY, '{"preferences":{"analytics":true}}']]);
   const writes: string[] = [];
   const scripts: Record<string, string>[] = [];
@@ -34,6 +37,7 @@ function fakeBrowser(navigator: AnalyticsEnv["navigator"] = {}) {
       },
     },
     navigator,
+    screen,
     addScript(attributes, onload) {
       scripts.push(attributes);
       onloads.push(onload);
@@ -129,6 +133,43 @@ describe("Do Not Track", () => {
   });
 });
 
+describe("a program rather than a person", () => {
+  const HEADLESS =
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/140.0.0.0 Safari/537.36";
+
+  test.each([
+    ["a browser driven by automation", fakeBrowser({ webdriver: true })],
+    ["a headless browser", fakeBrowser({ userAgent: HEADLESS })],
+    ["a bot", fakeBrowser({ userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1)" })],
+    ["a screen of exactly 800x600", fakeBrowser({}, { width: 800, height: 600 })],
+    ["a screen of exactly 1024x1024", fakeBrowser({}, { width: 1024, height: 1024 })],
+  ])("%s gets no script and sends no events", (_, browser) => {
+    const analytics = createAnalytics(CONFIG, browser.env);
+    expect(analytics.automated()).toBe(true);
+    expect(analytics.allowed()).toBe(false);
+    // Not Do Not Track: the page does not tell a program its browser sends it.
+    expect(analytics.blocked()).toBe(false);
+    analytics.init();
+    expect(analytics.track("search", { query: "scan", results: 4 })).toBe(false);
+    browser.runScripts();
+    expect(browser.scripts).toEqual([]);
+    expect(browser.events).toEqual([]);
+  });
+
+  test("a person's browser is not one", () => {
+    const analytics = createAnalytics(
+      CONFIG,
+      fakeBrowser({
+        webdriver: false,
+        userAgent:
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+      }).env,
+    );
+    expect(analytics.automated()).toBe(false);
+    expect(analytics.allowed()).toBe(true);
+  });
+});
+
 describe("the build's website id", () => {
   test("is the docs' own by default", () => {
     expect(umamiWebsiteId({})).toBe(DEFAULT_UMAMI_WEBSITE_ID);
@@ -219,4 +260,9 @@ describe("the privacy page", () => {
   test("names Do Not Track as what stops the counting", () => {
     expect(page).toContain("If your browser sends Do Not Track, the Umami script is not loaded");
   });
+
+  test("says automated visitors are not counted", () => {
+    expect(page).toContain("## Automated visitors");
+    expect(page).toContain("exactly 800 by 600 or 1024 by 1024");
+  });
 });

test/automated.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, test } from "bun:test";
+import { isAutomated, type Visitor } from "../src/lib/automated";
+
+// The rule in src/lib/automated.ts, on the visitors Umami counted and on people.
+
+const CHROME_WINDOWS =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
+const EDGE_WINDOWS = `${CHROME_WINDOWS} Edg/140.0.0.0`;
+const SAFARI_MAC =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15";
+const FIREFOX_LINUX = "Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Gecko/20100101 Firefox/143.0";
+const IPHONE =
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1";
+const CUBOT_ANDROID =
+  "Mozilla/5.0 (Linux; Android 13; CUBOT_X50) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Mobile Safari/537.36";
+
+const person = (userAgent: string, width = 1920, height = 1080): Visitor => ({
+  webdriver: false,
+  userAgent,
+  screen: { width, height },
+});
+
+describe("a person is counted", () => {
+  test.each([
+    ["Chrome on Windows", person(CHROME_WINDOWS)],
+    ["Edge on Windows", person(EDGE_WINDOWS, 1536, 864)],
+    ["Safari on a Mac", person(SAFARI_MAC, 1512, 982)],
+    ["Firefox on Linux", person(FIREFOX_LINUX, 2560, 1440)],
+    ["Safari on an iPhone", person(IPHONE, 393, 852)],
+    ["Chrome on a Cubot phone", person(CUBOT_ANDROID, 412, 915)],
+    ["a screen of 600x800, turned", person(CHROME_WINDOWS, 600, 800)],
+    ["a screen of 1024x768", person(CHROME_WINDOWS, 1024, 768)],
+  ])("%s", (_, visitor) => {
+    expect(isAutomated(visitor)).toBe(false);
+  });
+});
+
+describe("a program is not counted", () => {
+  test("a browser driven by automation", () => {
+    expect(isAutomated({ ...person(CHROME_WINDOWS), webdriver: true })).toBe(true);
+  });
+
+  test.each([
+    ["Googlebot", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"],
+    ["Bingbot", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"],
+    ["Applebot", `${SAFARI_MAC} (Applebot/0.1; +http://www.apple.com/go/applebot)`],
+    ["AhrefsBot", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"],
+    ["DuckDuckBot", "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)"],
+    [
+      "Yahoo Slurp",
+      "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+    ],
+    ["a crawler", "Mozilla/5.0 (compatible; SomeCrawler/1.0)"],
+    ["a spider", "Baiduspider+(+http://www.baidu.com/search/spider.htm)"],
+    ["GPTBot", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2)"],
+    ["ClaudeBot", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0)"],
+    ["headless Chrome", CHROME_WINDOWS.replace("Chrome/", "HeadlessChrome/")],
+    ["PhantomJS", "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 PhantomJS/2.1.1"],
+    ["Lighthouse", `${CHROME_WINDOWS} Chrome-Lighthouse`],
+    ["Slack's link preview", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"],
+    ["Twitter's card", "Twitterbot/1.0"],
+    [
+      "LinkedIn's preview",
+      "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)",
+    ],
+    ["Discord's embed", "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"],
+    ["Telegram's preview", "TelegramBot (like TwitterBot)"],
+    [
+      "Facebook's preview",
+      "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
+    ],
+    ["WhatsApp's preview", "WhatsApp/2.23.20.0 A"],
+    ["Skype's preview", "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"],
+    ["Bing's preview", `${CHROME_WINDOWS} BingPreview/1.0b`],
+    [
+      "Google's page renderer",
+      `${CHROME_WINDOWS} Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)`,
+    ],
+    ["Embedly", "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)"],
+    ["Iframely", "Iframely/1.3.1 (+https://iframely.com/docs/about)"],
+  ])("%s", (_, userAgent) => {
+    expect(isAutomated(person(userAgent))).toBe(true);
+  });
+
+  test.each([
+    ["Chrome on Windows at 800x600", person(CHROME_WINDOWS, 800, 600)],
+    ["Edge on Windows at 800x600", person(EDGE_WINDOWS, 800, 600)],
+    ["Safari on a Mac at 800x600", person(SAFARI_MAC, 800, 600)],
+    ["Chrome on Windows at 1024x1024", person(CHROME_WINDOWS, 1024, 1024)],
+  ])("%s", (_, visitor) => {
+    expect(isAutomated(visitor)).toBe(true);
+  });
+});