From 14a32d14d0f2d9cd1f0c13bbcdf4f89577ef392f Mon Sep 17 00:00:00 2001 From: Abdel Fane Date: Wed, 22 Jul 2026 16:45:39 -0600 Subject: [PATCH] feat(summary): gross downloads metric + close the summary-vs-dashboard gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `total.downloads` (and exCrypto variant): gross download events across npm + PyPI + Docker pulls + git clones + HF model downloads, counted the way the ecosystems themselves report downloads (incl. CI and re-downloads). This is the headline metric for the website; `adoption` stays the deduplicated floor. Views are never included. Also fixes the two pre-existing reconciliation bugs that made summary.json disagree with the dashboard by ~13.7K: - Docker was MAX(pull_count) across all rows — took only the single biggest image (8,378) instead of summing latest-per-image (18,544) - HuggingFace model downloads were omitted entirely (3,501) summary.total.adoption now equals the dashboard's combined.totalAdoption exactly (163,775). Gross downloads: 298,844. --- data/summary.json | 14 +++++++---- scripts/generate-summary.js | 46 +++++++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/data/summary.json b/data/summary.json index 3c5435cb..cae4f33b 100644 --- a/data/summary.json +++ b/data/summary.json @@ -1,13 +1,16 @@ { - "lastUpdated": "2026-07-22T20:21:16.853Z", + "lastUpdated": "2026-07-22T22:45:00.700Z", "total": { - "adoption": 150108, - "adoptionExCrypto": 132929, + "adoption": 163775, + "adoptionExCrypto": 146596, + "downloads": 298844, + "downloadsExCrypto": 278460, "npm": 112222, "pypi": 28487, "cloneUniques": 1021, "clones": 136090, - "docker": 8378, + "docker": 18544, + "hf": 3501, "views": 28164, "stars": 248, "repos": 45, @@ -30,6 +33,7 @@ "pypi": 12784, "cloneUniques": 1012, "clones": 132876, - "docker": 8378 + "docker": 18544, + "downloads": 278460 } } \ No newline at end of file diff --git a/scripts/generate-summary.js b/scripts/generate-summary.js index 68744746..46437284 100644 --- a/scripts/generate-summary.js +++ b/scripts/generate-summary.js @@ -58,11 +58,31 @@ try { ORDER BY ts.date DESC LIMIT 1 `).get(); - // Docker totals + // Docker totals: latest pull_count per image, summed across images — matching + // the dashboard (pages/api/overview.js). MAX() across all rows silently took + // only the single biggest image and undercounted the fleet by ~10K. const dockerTotal = db.prepare(` - SELECT COALESCE(MAX(pull_count), 0) as total FROM docker_pulls + SELECT COALESCE(SUM(p), 0) AS total FROM ( + SELECT (SELECT pull_count FROM docker_pulls dp2 + WHERE dp2.image_id = dp.image_id ORDER BY date DESC LIMIT 1) AS p + FROM docker_pulls dp GROUP BY dp.image_id + ) `).get(); + // HuggingFace model downloads: latest all-time count per model, summed — + // matching the dashboard. Previously omitted here entirely, which was the + // other half of the summary-vs-dashboard reconciliation gap. + const hfTableExists = db.prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='huggingface_stats'" + ).get(); + const hfTotal = hfTableExists ? db.prepare(` + SELECT COALESCE(SUM(d), 0) AS total FROM ( + SELECT (SELECT downloads_all_time FROM huggingface_stats h2 + WHERE h2.model_id = h.model_id ORDER BY date DESC LIMIT 1) AS d + FROM huggingface_stats h GROUP BY h.model_id + ) + `).get() : { total: 0 }; + // GitHub views const viewsTotal = db.prepare(` SELECT COALESCE(SUM(count), 0) as total FROM traffic_views @@ -139,14 +159,29 @@ try { const npmPkgCount = db.prepare('SELECT COUNT(*) as count FROM npm_packages').get(); // Total adoption (all ecosystems). Clone term = UNIQUE CLONERS, not raw count. - const totalAll = npmTotal.total + pypiTotal.total + cloneUniquesTotal.total + dockerTotal.total; - const totalExCrypto = npmExCrypto.total + pypiExCrypto.total + cloneUniquesExCrypto.total + dockerTotal.total; + // Same formula as the dashboard's combined.totalAdoption (overview.js): + // uniqueCloners + npm + pypi + docker + hf — the two surfaces must agree. + const totalAll = npmTotal.total + pypiTotal.total + cloneUniquesTotal.total + dockerTotal.total + hfTotal.total; + const totalExCrypto = npmExCrypto.total + pypiExCrypto.total + cloneUniquesExCrypto.total + dockerTotal.total + hfTotal.total; + + // Gross DOWNLOADS (all ecosystems): npm downloads + PyPI downloads + Docker + // pulls + git clones (raw) + HF model downloads. Every term is literally a + // download event, counted the same way the ecosystems themselves report + // downloads (incl. CI and re-downloads — same semantics as npm's public + // download counts). This is the marketing/headline metric; `adoption` above + // stays the deduplicated floor. Views are NOT downloads, never included. + const downloadsAll = npmTotal.total + pypiTotal.total + clonesTotal.total + dockerTotal.total + hfTotal.total; + const downloadsExCrypto = npmExCrypto.total + pypiExCrypto.total + clonesExCrypto.total + dockerTotal.total + hfTotal.total; const summary = { lastUpdated: new Date().toISOString(), total: { adoption: totalAll, adoptionExCrypto: totalExCrypto, + // Gross download events across npm + PyPI + Docker + git clones. The + // website's "downloads" headline sources THIS field, never `adoption`. + downloads: downloadsAll, + downloadsExCrypto, npm: npmTotal.total, pypi: pypiTotal.total, // Distinct cloners — the term summed into `adoption`. @@ -154,6 +189,7 @@ try { // Raw git-clone count — context/transparency only, NOT in `adoption`. clones: clonesTotal.total, docker: dockerTotal.total, + hf: hfTotal.total, views: viewsTotal.total, stars: starsTotal.total, repos: repoCount.count, @@ -171,6 +207,7 @@ try { cloneUniques: cloneUniquesExCrypto.total, clones: clonesExCrypto.total, docker: dockerTotal.total, + downloads: downloadsExCrypto, }, }; @@ -179,6 +216,7 @@ try { console.log(`Summary written to ${outPath}`); console.log(` Total adoption: ${totalAll.toLocaleString()}`); console.log(` Excl. CryptoServe: ${totalExCrypto.toLocaleString()}`); + console.log(` Gross downloads: ${downloadsAll.toLocaleString()} (excl. CryptoServe: ${downloadsExCrypto.toLocaleString()})`); } catch (err) { console.error('Error generating summary:', err); process.exit(1);