diff --git a/data/summary.json b/data/summary.json index 63f2177c..3c5435cb 100644 --- a/data/summary.json +++ b/data/summary.json @@ -1,10 +1,11 @@ { - "lastUpdated": "2026-07-22T06:49:31.191Z", + "lastUpdated": "2026-07-22T20:21:16.853Z", "total": { - "adoption": 285177, - "adoptionExCrypto": 264793, + "adoption": 150108, + "adoptionExCrypto": 132929, "npm": 112222, "pypi": 28487, + "cloneUniques": 1021, "clones": 136090, "docker": 8378, "views": 28164, @@ -27,6 +28,7 @@ "excludingCrypto": { "npm": 110755, "pypi": 12784, + "cloneUniques": 1012, "clones": 132876, "docker": 8378 } diff --git a/lib/adoption.js b/lib/adoption.js new file mode 100644 index 00000000..68fb8bb3 --- /dev/null +++ b/lib/adoption.js @@ -0,0 +1,71 @@ +/** + * The one definition of "adoption" — shared by the API (pages/api/overview.js, + * server) and the dashboard (pages/index.js, client) so the number can never + * drift between where it's computed and where it's shown. + * + * Adoption sums acquisition events: distinct code-pullers + package installs + + * image pulls + model downloads. The clone term is UNIQUE CLONERS, never the raw + * git-clone count. + * + * Why uniques and not raw clones: GitHub's clone `count` is dominated by + * automation — a single CI/mirror/scanner re-cloning a repo generates thousands + * of clones from a handful of actors (hackmyagent: ~2,000 clones/day from 5-10 + * unique cloners, a 147x re-clone ratio). Raw count answers "how many git-clone + * operations happened", which is not adoption. Unique cloners answers "how many + * distinct people/systems pulled the code", which is. Raw clone count is retained + * on the GitHub-traffic surfaces as an operational metric, never in adoption. + * + * What "unique cloners" is HERE: GitHub's own 14-day DEDUPLICATED distinct-cloner + * count (`traffic_summary.clones_uniques`, surfaced as `recentUniqueCloners`). + * That is the only honest distinct-cloner figure GitHub exposes — you cannot + * deduplicate cloners across days you never captured identity for, so there is no + * true 30-day or all-time unique-cloner count to be had. It is therefore a + * ROLLING 14-DAY SNAPSHOT (like Chrome weekly-active users or the 14-day unique + * visitors), shown identically under every period rather than re-windowed. + * + * Deliberately NOT used: SUM(traffic_clones.uniques) over a window. That sums + * GitHub's per-day distinct-cloner counts, i.e. cloner-DAYS — a cloner active on + * N days counts N times — which re-inflates the exact signal this fix removes and + * would be a number labeled "unique cloners" that is not one. + */ + +// Pick the value for the selected period from an object whose fields are named +// per-window (all / 30d / 7d / 24h / custom). Shared shape across github/npm/pypi. +function periodPick(o, allKey, k30, k7, k24, period, kCustom) { + if (!o) return 0; + switch (period) { + case '24h': return o[k24] || 0; + case '7d': return o[k7] || 0; + case '30d': return o[k30] || 0; + case 'custom': return o[kCustom] || 0; + default: return o[allKey] || 0; + } +} + +// HuggingFace only reports all-time + rolling 30d, nothing finer. +function hfPeriod(hf, period) { + if (!hf) return null; + if (period === 'all') return hf.downloadsAllTime || 0; + if (period === '30d') return hf.downloads30d || 0; + return null; // 24h / 7d / custom -> not measured at this granularity +} + +// The invariant: adoption is cloners + installs, never raw clone count. Every +// surface that reports adoption funnels through here. +function sumAdoption({ cloneUniques = 0, npm = 0, pypi = 0, docker = 0, hf = 0 }) { + return cloneUniques + npm + pypi + docker + hf; +} + +// Period-aware adoption for one product row (used by the dashboard table + KPIs). +// The clone term is the 14-day deduplicated unique-cloner SNAPSHOT (period- +// independent — see the module header), NOT raw clones and NOT summed cloner-days. +function rowAdoption(p, period) { + const cloneUniques = p.github?.cloneUniques || 0; // 14-day deduped snapshot + const npm = periodPick(p.npm, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'); + const pypi = periodPick(p.pypi, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'); + const docker = period === 'all' ? (p.docker?.totalPulls || 0) : 0; + const hf = hfPeriod(p.hf, period) || 0; + return sumAdoption({ cloneUniques, npm, pypi, docker, hf }); +} + +module.exports = { periodPick, hfPeriod, sumAdoption, rowAdoption }; diff --git a/lib/repos.js b/lib/repos.js index 1a2cff15..09652efe 100644 --- a/lib/repos.js +++ b/lib/repos.js @@ -82,7 +82,8 @@ function mergeByCanonical(repoStats) { totalClones: sum('totalClones'), clones24h: sum('clones24h'), clones7d: sum('clones7d'), clones30d: sum('clones30d'), customClones: sum('customClones'), - // 14-day deduplicated uniques are a snapshot — take the canonical (live) one. + // 14-day deduplicated uniques are a snapshot (the adoption clone term) — take + // the canonical (live) row's value, never summed across twins. recentUniqueVisitors: c.recentUniqueVisitors || 0, recentUniqueCloners: c.recentUniqueCloners || 0, // Snapshot metrics — from the canonical row, never summed. @@ -117,21 +118,30 @@ function canonicalRepoTotals(db) { const latestFork = db.prepare( 'SELECT total_forks FROM forks WHERE repo_id = ? ORDER BY date DESC LIMIT 1' ); + // Latest 14-day deduplicated distinct-cloner count per repo (a snapshot). + const latestUniqCloners = db.prepare( + 'SELECT clones_uniques FROM traffic_summary WHERE repo_id = ? ORDER BY date DESC LIMIT 1' + ); const rows = repos.map(r => ({ full_name: r.full_name, canonical_full_name: r.canonical_full_name, stars: latestStar.get(r.id)?.total_stars || 0, forks: latestFork.get(r.id)?.total_forks || 0, + uniqueCloners: latestUniqCloners.get(r.id)?.clones_uniques || 0, })); - let stars = 0, forks = 0, repoCount = 0; + let stars = 0, forks = 0, repoCount = 0, uniqueCloners = 0; for (const [canon, group] of groupByCanonical(rows)) { const c = pickCanonical(group, canon); stars += c.stars || 0; forks += c.forks || 0; + // 14-day dedup snapshot from the canonical (live) row — NEVER summed across + // twins (you cannot add two rolling 14-day dedup counts). Stale slug's own + // clones_uniques is dropped, matching mergeByCanonical / the overview API. + uniqueCloners += c.uniqueCloners || 0; repoCount += 1; } - return { stars, forks, repoCount }; + return { stars, forks, repoCount, uniqueCloners }; } module.exports = { diff --git a/lib/standards.js b/lib/standards.js index abcd727a..3c1dd9e2 100644 --- a/lib/standards.js +++ b/lib/standards.js @@ -59,6 +59,8 @@ function mergeSpec(rows) { github: { views: sum('totalViews'), views24h: sum('views24h'), views7d: sum('views7d'), views30d: sum('views30d'), customViews: sum('customViews'), clones: sum('totalClones'), clones24h: sum('clones24h'), clones7d: sum('clones7d'), clones30d: sum('clones30d'), customClones: sum('customClones'), + // 14-day deduplicated distinct cloners (snapshot) — from the canonical row, not summed. + cloneUniques: canonical.recentUniqueCloners || 0, stars: canonical.stars || 0, starsGrowth24h: canonical.starsGrowth24h || 0, starsGrowth7d: canonical.starsGrowth7d || 0, starsGrowth30d: canonical.starsGrowth30d || 0, starsGrowthAll: canonical.starsGrowthAll || 0, starsGrowthCustom: canonical.starsGrowthCustom || 0, @@ -73,6 +75,7 @@ function sumGithub(members) { return { views: acc('views'), views24h: acc('views24h'), views7d: acc('views7d'), views30d: acc('views30d'), customViews: acc('customViews'), clones: acc('clones'), clones24h: acc('clones24h'), clones7d: acc('clones7d'), clones30d: acc('clones30d'), customClones: acc('customClones'), + cloneUniques: acc('cloneUniques'), stars: acc('stars'), starsGrowth24h: acc('starsGrowth24h'), starsGrowth7d: acc('starsGrowth7d'), starsGrowth30d: acc('starsGrowth30d'), starsGrowthAll: acc('starsGrowthAll'), starsGrowthCustom: acc('starsGrowthCustom'), diff --git a/pages/api/overview.js b/pages/api/overview.js index 9d8f3cae..916ca917 100644 --- a/pages/api/overview.js +++ b/pages/api/overview.js @@ -3,6 +3,7 @@ const path = require('path'); const { computeWindowedStats, computeStarGrowth } = require('../../lib/windowing'); const { groupStandards } = require('../../lib/standards'); const { mergeByCanonical } = require('../../lib/repos'); +const { sumAdoption } = require('../../lib/adoption'); /** * Overview API: Returns combined GitHub + npm metrics suitable for @@ -74,6 +75,12 @@ export default async function handler(req, res) { FROM traffic_clones WHERE repo_id = ? AND date > date(${clonesAnchor}, '-30 days') `).get(repo.id); + // Unique cloners — the adoption signal (distinct pullers) — is GitHub's own + // 14-day DEDUPLICATED count (traffic_summary.clones_uniques, read into + // recentUniqueCloners below), NOT a sum of per-day uniques. See lib/adoption.js: + // summing daily uniques double-counts a cloner across days and re-inflates the + // exact noise this replaces. GitHub only dedupes over 14 days, so this is a + // rolling snapshot, handled like recentUniqueVisitors / Chrome users. const summary = db.prepare(` SELECT views_uniques, clones_uniques FROM traffic_summary WHERE repo_id = ? @@ -88,7 +95,9 @@ export default async function handler(req, res) { ORDER BY date DESC LIMIT 1 `).get(repo.id); - // Custom date range queries for GitHub traffic + // Custom date range queries for GitHub traffic. (Unique cloners has no + // custom-range form — GitHub only dedupes over a rolling 14 days, so the + // snapshot in recentUniqueCloners is used for every window.) let customViews = 0, customClones = 0; if (isCustomRange) { const cv = db.prepare(` @@ -118,6 +127,7 @@ export default async function handler(req, res) { clones30d: clones30d.total, customClones, recentUniqueVisitors: summary?.views_uniques || 0, + // 14-day deduplicated distinct cloners — the adoption clone term. See lib/adoption.js. recentUniqueCloners: summary?.clones_uniques || 0, ...starGrowth, forks: forks?.forks || 0, @@ -362,6 +372,8 @@ export default async function handler(req, res) { const matchedChrome = chromeStats.filter(c => (config.chromeExtensions || []).includes(c.extensionId)); const githubClones = matchedRepos.reduce((s, r) => s + r.totalClones, 0); + // 14-day deduplicated distinct cloners across the tool's repos (snapshot). + const githubCloneUniques = matchedRepos.reduce((s, r) => s + (r.recentUniqueCloners || 0), 0); const npmDownloads = matchedPackages.reduce((s, p) => s + p.allTimeDownloads, 0); const pypiDownloads = matchedPypi.reduce((s, p) => s + p.allTimeDownloads, 0); const dockerPulls = matchedDocker.reduce((s, d) => s + d.totalPulls, 0); @@ -381,6 +393,9 @@ export default async function handler(req, res) { clones7d: matchedRepos.reduce((s, r) => s + r.clones7d, 0), clones30d: matchedRepos.reduce((s, r) => s + r.clones30d, 0), customClones: matchedRepos.reduce((s, r) => s + (r.customClones || 0), 0), + // 14-day deduplicated distinct cloners (snapshot) — the clone term used + // in adoption. Period-independent; see lib/adoption.js. + cloneUniques: githubCloneUniques, stars: matchedRepos.reduce((s, r) => s + r.stars, 0), starsGrowth24h: matchedRepos.reduce((s, r) => s + (r.starsGrowth24h || 0), 0), starsGrowth7d: matchedRepos.reduce((s, r) => s + (r.starsGrowth7d || 0), 0), @@ -429,9 +444,11 @@ export default async function handler(req, res) { ratingCount: matchedChrome[0]?.ratingCount ?? null, extensionCount: matchedChrome.length, }, - // Combined adoption: clones + npm + pypi + docker pulls + HF model downloads. - // Chrome weekly-active users are intentionally excluded (snapshot, not installs). - totalAdoption: githubClones + npmDownloads + pypiDownloads + dockerPulls + hfDownloads, + // Combined adoption: UNIQUE CLONERS + npm + pypi + docker pulls + HF model + // downloads. Raw clone count is deliberately NOT used — it is CI/mirror + // re-clone noise (see lib/adoption.js). Chrome weekly-active users are + // excluded too (a snapshot, not installs). + totalAdoption: sumAdoption({ cloneUniques: githubCloneUniques, npm: npmDownloads, pypi: pypiDownloads, docker: dockerPulls, hf: hfDownloads }), }; }); @@ -465,6 +482,7 @@ export default async function handler(req, res) { if (ungroupedRepos.length > 0 || ungroupedNpm.length > 0 || ungroupedPypi.length > 0 || ungroupedDocker.length > 0 || ungroupedHf.length > 0) { const ugClones = ungroupedRepos.reduce((s, r) => s + r.totalClones, 0); + const ugCloneUniques = ungroupedRepos.reduce((s, r) => s + (r.recentUniqueCloners || 0), 0); const ugNpm = ungroupedNpm.reduce((s, p) => s + p.allTimeDownloads, 0); const ugPypi = ungroupedPypi.reduce((s, p) => s + p.allTimeDownloads, 0); const ugDocker = ungroupedDocker.reduce((s, d) => s + d.totalPulls, 0); @@ -483,6 +501,7 @@ export default async function handler(req, res) { clones7d: ungroupedRepos.reduce((s, r) => s + r.clones7d, 0), clones30d: ungroupedRepos.reduce((s, r) => s + r.clones30d, 0), customClones: ungroupedRepos.reduce((s, r) => s + (r.customClones || 0), 0), + cloneUniques: ugCloneUniques, stars: ungroupedRepos.reduce((s, r) => s + r.stars, 0), starsGrowth24h: ungroupedRepos.reduce((s, r) => s + (r.starsGrowth24h || 0), 0), starsGrowth7d: ungroupedRepos.reduce((s, r) => s + (r.starsGrowth7d || 0), 0), @@ -522,7 +541,7 @@ export default async function handler(req, res) { usersGrowth7d: 0, usersGrowth30d: 0, rating: null, ratingCount: null, extensionCount: ungroupedChrome.length, }, - totalAdoption: ugClones + ugNpm + ugPypi + ugDocker + ugHf, + totalAdoption: sumAdoption({ cloneUniques: ugCloneUniques, npm: ugNpm, pypi: ugPypi, docker: ugDocker, hf: ugHf }), }); } @@ -542,6 +561,8 @@ export default async function handler(req, res) { clones24h: repoStats.reduce((s, r) => s + r.clones24h, 0), clones7d: repoStats.reduce((s, r) => s + r.clones7d, 0), clones30d: repoStats.reduce((s, r) => s + r.clones30d, 0), + // 14-day deduplicated distinct cloners across all repos (snapshot). + totalCloneUniques: repoStats.reduce((s, r) => s + (r.recentUniqueCloners || 0), 0), totalStars: repoStats.reduce((s, r) => s + r.stars, 0), starsGrowth24h: repoStats.reduce((s, r) => s + (r.starsGrowth24h || 0), 0), starsGrowth7d: repoStats.reduce((s, r) => s + (r.starsGrowth7d || 0), 0), @@ -583,11 +604,14 @@ export default async function handler(req, res) { usersGrowth30d: chromeStats.reduce((s, c) => s + (c.usersGrowth30d || 0), 0), }, combined: { - totalAdoption: repoStats.reduce((s, r) => s + r.totalClones, 0) + - npmStats.reduce((s, p) => s + p.allTimeDownloads, 0) + - totalPypiDownloads + - totalDockerPulls + - totalHfDownloads, + // 14-day deduplicated unique cloners, not raw clone count — see lib/adoption.js. + totalAdoption: sumAdoption({ + cloneUniques: repoStats.reduce((s, r) => s + (r.recentUniqueCloners || 0), 0), + npm: npmStats.reduce((s, p) => s + p.allTimeDownloads, 0), + pypi: totalPypiDownloads, + docker: totalDockerPulls, + hf: totalHfDownloads, + }), totalPageViews: repoStats.reduce((s, r) => s + r.totalViews, 0), }, }; @@ -678,6 +702,7 @@ export default async function handler(req, res) { stars: standards.reduce((s, f) => s + (f.github.stars || 0), 0), views: standards.reduce((s, f) => s + (f.github.views || 0), 0), clones: standards.reduce((s, f) => s + (f.github.clones || 0), 0), + cloneUniques: standards.reduce((s, f) => s + (f.github.cloneUniques || 0), 0), forks: standards.reduce((s, f) => s + (f.github.forks || 0), 0), }; diff --git a/pages/index.js b/pages/index.js index 42ed0d83..42905035 100644 --- a/pages/index.js +++ b/pages/index.js @@ -10,6 +10,9 @@ import { Package, TrendingUp, TrendingDown, Container, Activity, Github, Box, Brain, Menu, Search, ArrowUp, ArrowDown, FileText, Chrome, } from 'lucide-react'; +// Adoption = unique cloners + installs, defined once and shared with the API so +// the number can't drift between where it's computed and where it's shown. +import { periodPick, hfPeriod, rowAdoption } from '../lib/adoption'; /* ============================================================ Source identity — one place defines color + icon per source @@ -102,17 +105,8 @@ function weeklyDownloads(rows) { return Object.values(w).sort((a, b) => a.week.localeCompare(b.week)); } -/* Period picker for product rows (shared by table + KPI aggregation) */ -function periodPick(o, allKey, k30, k7, k24, period, kCustom) { - if (!o) return 0; - switch (period) { - case '24h': return o[k24] || 0; - case '7d': return o[k7] || 0; - case '30d': return o[k30] || 0; - case 'custom': return o[kCustom] || 0; - default: return o[allKey] || 0; - } -} +/* periodPick, hfPeriod and rowAdoption live in lib/adoption.js (imported above) + so the API and the dashboard share one definition of adoption. */ /* Star growth for the selected period. Stars are cumulative, so this is the count gained within the window (not a percentage). */ function starGrowthPick(g, period) { @@ -124,22 +118,6 @@ function starGrowthPick(g, period) { default: return g?.starsGrowthAll; } } -/* HuggingFace only reports all-time + rolling 30d, nothing finer */ -function hfPeriod(hf, period) { - if (!hf) return null; - if (period === 'all') return hf.downloadsAllTime || 0; - if (period === '30d') return hf.downloads30d || 0; - return null; // 24h / 7d / custom -> not measured at this granularity -} -function rowAdoption(p, period) { - const clones = periodPick(p.github, 'clones', 'clones30d', 'clones7d', 'clones24h', period, 'customClones'); - const npm = periodPick(p.npm, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'); - const pypi = periodPick(p.pypi, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'); - const docker = period === 'all' ? (p.docker?.totalPulls || 0) : 0; - const hf = hfPeriod(p.hf, period) || 0; - return clones + npm + pypi + docker + hf; -} - /* ============================================================ Primitive UI ============================================================ */ @@ -531,7 +509,9 @@ function OverviewTab({ overview, loading, trends, granularity, setGranularity, p const channelData = [ { name: 'npm', value: totals.npm?.allTimeDownloads || 0, color: C.npm }, - { name: 'Git Clones', value: totals.github?.totalClones || 0, color: C.github }, + // Unique cloners (GitHub's 14-day deduplicated count), not raw clone count — + // the raw count is CI/mirror re-clone noise, shown only on the GitHub tab. + { name: 'Unique Cloners (14d)', value: totals.github?.totalCloneUniques || 0, color: C.github }, { name: 'PyPI', value: totals.pypi?.allTimeDownloads || 0, color: C.pypi }, { name: 'Docker', value: totals.docker?.totalPulls || 0, color: C.docker }, { name: 'HF Models', value: totals.hf?.downloadsAllTime || 0, color: C.hf }, @@ -550,7 +530,7 @@ function OverviewTab({ overview, loading, trends, granularity, setGranularity, p
- Git clones + package installs + image pulls + model downloads across {totals.github.repos} repositories, + Unique cloners + package installs + image pulls + model downloads across {totals.github.repos} repositories, {' '}{(totals.npm?.packages || 0) + (totals.pypi?.packages || 0)} packages, {totals.docker?.images || 0} images and {totals.hf?.models || 0} models.
@@ -589,7 +569,7 @@ function OverviewTab({ overview, loading, trends, granularity, setGranularity, p } pad={false}> {(period === 'custom' && !custom) @@ -670,8 +650,9 @@ function OverviewTab({ overview, loading, trends, granularity, setGranularity, p
} rows={[ ['Repositories', totals.github.repos], ['Page views', totals.github.totalViews], - ['Git clones', totals.github.totalClones], ['Stars', totals.github.totalStars], - ['Forks', totals.github.totalForks], ['Contributors', totals.github.totalContributors || 0], + ['Unique cloners (14d)', totals.github.totalCloneUniques || 0], ['Git clones (raw)', totals.github.totalClones], + ['Stars', totals.github.totalStars], ['Forks', totals.github.totalForks], + ['Contributors', totals.github.totalContributors || 0], ]} /> } rows={[ ['Packages', totals.npm.packages], ['All-time installs', totals.npm.allTimeDownloads], @@ -833,7 +814,10 @@ function AdoptionTable({ products, period }) { const cols = [ { key: 'name', label: 'Tool', sortValue: r => r.name, render: r => (
{r.name}
{r.description}
) }, { key: 'views', label: 'Views', align: 'right', sortValue: r => periodPick(r.github, 'views', 'views30d', 'views7d', 'views24h', period, 'customViews'), render: r => fmtFull(periodPick(r.github, 'views', 'views30d', 'views7d', 'views24h', period, 'customViews')) }, - { key: 'clones', label: 'Clones', align: 'right', sortValue: r => periodPick(r.github, 'clones', 'clones30d', 'clones7d', 'clones24h', period, 'customClones'), render: r => fmtFull(periodPick(r.github, 'clones', 'clones30d', 'clones7d', 'clones24h', period, 'customClones')) }, + // Unique cloners (distinct pullers) = GitHub's 14-day deduplicated count, the + // clone term that feeds Adoption. A rolling 14d snapshot (period-independent); + // raw clone count lives on the GitHub tab. See lib/adoption.js. + { key: 'cloners', label: 'Cloners 14d', align: 'right', sortValue: r => r.github?.cloneUniques || 0, render: r => fmtFull(r.github?.cloneUniques || 0) }, { key: 'npm', label: 'npm', align: 'right', sortValue: r => periodPick(r.npm, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'), render: r => (<>{fmtFull(periodPick(r.npm, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'))}{(period === '30d' || period === '7d') && }), @@ -852,7 +836,7 @@ function AdoptionTable({ products, period }) { ]; const t = products.reduce((a, p) => { a.views += periodPick(p.github, 'views', 'views30d', 'views7d', 'views24h', period, 'customViews'); - a.clones += periodPick(p.github, 'clones', 'clones30d', 'clones7d', 'clones24h', period, 'customClones'); + a.clones += p.github?.cloneUniques || 0; // 14-day deduped snapshot (period-independent) a.npm += periodPick(p.npm, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'); a.pypi += periodPick(p.pypi, 'allTimeDownloads', 'last30Downloads', 'last7Downloads', 'last24hDownloads', period, 'customDownloads'); a.docker += period === 'all' ? (p.docker?.totalPulls || 0) : 0; @@ -881,7 +865,7 @@ function AdoptionTable({ products, period }) { /* ============================================================ Standards — specs, protocols & conformance suites These are specifications, not installable packages, so adoption - is read as stars / views / clones, not downloads. + is read as stars / views / unique cloners, not downloads. ============================================================ */ function StandardsTab({ overview, loading, period }) { const [filter, setFilter] = useState(''); @@ -892,7 +876,8 @@ function StandardsTab({ overview, loading, period }) { const periodWord = PERIOD_LABEL[period]; const totals = overview?.totals?.standards || {}; const pickViews = g => periodPick(g, 'views', 'views30d', 'views7d', 'views24h', period, 'customViews'); - const pickClones = g => periodPick(g, 'clones', 'clones30d', 'clones7d', 'clones24h', period, 'customClones'); + // 14-day deduplicated distinct cloners (snapshot), consistent with tool adoption. + const pickClones = g => g?.cloneUniques || 0; const agg = standards.reduce((a, f) => { a.views += pickViews(f.github); @@ -908,7 +893,7 @@ function StandardsTab({ overview, loading, period }) { { key: 'repoCount', label: 'Specs', align: 'right', sortValue: r => r.repoCount, render: r => fmtFull(r.repoCount) }, { key: 'stars', label: 'Stars', align: 'right', sortValue: r => r.github?.stars || 0, render: r => starCell(r.github) }, { key: 'views', label: 'Views', align: 'right', sortValue: r => pickViews(r.github), render: r => fmtFull(pickViews(r.github)) }, - { key: 'clones', label: 'Clones', align: 'right', lead: true, sortValue: r => pickClones(r.github), render: r => fmtFull(pickClones(r.github)) }, + { key: 'cloners', label: 'Cloners 14d', align: 'right', lead: true, sortValue: r => pickClones(r.github), render: r => fmtFull(pickClones(r.github)) }, ]; const famFooter = [ Total, @@ -926,7 +911,7 @@ function StandardsTab({ overview, loading, period }) { { key: 'name', label: 'Repository', sortValue: r => r.name, render: r => (
{r.name}
{r.family}
) }, { key: 'stars', label: 'Stars', align: 'right', sortValue: r => r.github?.stars || 0, render: r => starCell(r.github) }, { key: 'views', label: 'Views', align: 'right', sortValue: r => pickViews(r.github), render: r => fmtFull(pickViews(r.github)) }, - { key: 'clones', label: 'Clones', align: 'right', lead: true, sortValue: r => pickClones(r.github), render: r => fmtFull(pickClones(r.github)) }, + { key: 'cloners', label: 'Cloners 14d', align: 'right', lead: true, sortValue: r => pickClones(r.github), render: r => fmtFull(pickClones(r.github)) }, ]; return ( @@ -935,10 +920,10 @@ function StandardsTab({ overview, loading, period }) { } color={C.standards} label="Specifications" value={fmtFull(totals.repos || 0)} sub={`${standards.length} families`} /> } color={C.pypi} label="Stars" value={fmtFull(agg.stars)} sub={agg.starsGrowth > 0 ? `+${fmtFull(agg.starsGrowth)} ${periodWord}` : 'current total'} /> } color={C.github} label="Views" value={fmtFull(agg.views)} sub={periodWord} /> - } color={C.accent} label="Clones" value={fmtFull(agg.clones)} sub={periodWord} /> + } color={C.accent} label="Cloners" value={fmtFull(agg.clones)} sub="unique · last 14d" />
- + r.name} initialSort={{ key: 'stars', dir: 'desc' }} footer={famFooter} /> diff --git a/scripts/generate-summary.js b/scripts/generate-summary.js index c4ec68d3..68744746 100644 --- a/scripts/generate-summary.js +++ b/scripts/generate-summary.js @@ -36,7 +36,10 @@ try { WHERE p.name NOT LIKE 'cryptoserve%' `).get(); - // GitHub clones totals + // GitHub clones. Adoption counts UNIQUE CLONERS (distinct pullers), NOT the raw + // clone count — raw count is dominated by CI/mirror re-cloning (a handful of + // actors looping) and is not adoption. See lib/adoption.js. The raw count is + // still surfaced as `clones` for context/transparency. const clonesTotal = db.prepare(` SELECT COALESCE(SUM(count), 0) as total FROM traffic_clones `).get(); @@ -47,6 +50,14 @@ try { WHERE r.full_name != 'ecolibria/cryptoserve' `).get(); + // cryptoserve's own 14-day deduped cloners, to subtract for the ex-crypto total. + const cryptoUniqueCloners = db.prepare(` + SELECT COALESCE(clones_uniques, 0) AS u + FROM traffic_summary ts JOIN repositories r ON r.id = ts.repo_id + WHERE r.full_name = 'ecolibria/cryptoserve' + ORDER BY ts.date DESC LIMIT 1 + `).get(); + // Docker totals const dockerTotal = db.prepare(` SELECT COALESCE(MAX(pull_count), 0) as total FROM docker_pulls @@ -66,6 +77,13 @@ try { const starsTotal = { total: canonTotals.stars }; const repoCount = { count: canonTotals.repoCount }; + // Unique cloners = 14-day deduplicated distinct cloners, deduped by canonical + // repo (twins collapsed, snapshot from the canonical row) — identical to the + // dashboard's totals.github.totalCloneUniques. NOT SUM(daily uniques) and NOT a + // sum across twin rows, both of which over-count. See lib/adoption.js. + const cloneUniquesTotal = { total: canonTotals.uniqueCloners }; + const cloneUniquesExCrypto = { total: canonTotals.uniqueCloners - (cryptoUniqueCloners?.u || 0) }; + // Chrome Web Store weekly-active users (latest snapshot per extension, // summed across extensions). This is NOT a download count and is deliberately // excluded from total adoption — surfaced separately for context. @@ -120,9 +138,9 @@ try { // npm package count const npmPkgCount = db.prepare('SELECT COUNT(*) as count FROM npm_packages').get(); - // Total adoption (all ecosystems) - const totalAll = npmTotal.total + pypiTotal.total + clonesTotal.total + dockerTotal.total; - const totalExCrypto = npmExCrypto.total + pypiExCrypto.total + clonesExCrypto.total + dockerTotal.total; + // Total adoption (all ecosystems). Clone term = UNIQUE CLONERS, not raw count. + const totalAll = npmTotal.total + pypiTotal.total + cloneUniquesTotal.total + dockerTotal.total; + const totalExCrypto = npmExCrypto.total + pypiExCrypto.total + cloneUniquesExCrypto.total + dockerTotal.total; const summary = { lastUpdated: new Date().toISOString(), @@ -131,6 +149,9 @@ try { adoptionExCrypto: totalExCrypto, npm: npmTotal.total, pypi: pypiTotal.total, + // Distinct cloners — the term summed into `adoption`. + cloneUniques: cloneUniquesTotal.total, + // Raw git-clone count — context/transparency only, NOT in `adoption`. clones: clonesTotal.total, docker: dockerTotal.total, views: viewsTotal.total, @@ -147,6 +168,7 @@ try { excludingCrypto: { npm: npmExCrypto.total, pypi: pypiExCrypto.total, + cloneUniques: cloneUniquesExCrypto.total, clones: clonesExCrypto.total, docker: dockerTotal.total, }, diff --git a/test/adoption.test.js b/test/adoption.test.js new file mode 100644 index 00000000..0a653ad0 --- /dev/null +++ b/test/adoption.test.js @@ -0,0 +1,82 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { periodPick, hfPeriod, sumAdoption, rowAdoption } = require('../lib/adoption'); + +/* + * The whole point of this module: adoption counts UNIQUE CLONERS, never raw + * git-clone count. "Unique cloners" here is GitHub's 14-day DEDUPLICATED count + * (recentUniqueCloners / clones_uniques) — a period-independent snapshot — NOT a + * sum of per-day uniques (which would count a cloner once per active day and + * re-inflate the very noise this removes). hackmyagent: ~71,000 raw clones and + * ~2,000/day from 5-10 actors, but only ~69 distinct cloners over 14 days. + * If a future edit wires raw clones (or cloner-days) back into adoption, these fail. + */ + +// A product whose raw clone count is enormous but whose deduped unique-cloner +// count is tiny — the exact hackmyagent shape. +function hmaShaped() { + return { + github: { + clones: 71077, clones30d: 31231, clones7d: 14000, clones24h: 2135, customClones: 999, + cloneUniques: 69, // 14-day deduplicated distinct cloners (snapshot) + }, + npm: { allTimeDownloads: 27731, last30Downloads: 1976, last7Downloads: 400, last24hDownloads: 50, customDownloads: 12 }, + pypi: {}, docker: { totalPulls: 0 }, hf: null, + }; +} + +test('adoption uses deduped unique cloners, NOT raw clone count (30d)', () => { + const p = hmaShaped(); + // 30d: cloneUniques(69) + npm last30(1976) = 2045. Raw 31231 must not appear. + assert.equal(rowAdoption(p, '30d'), 69 + 1976); + assert.notEqual(rowAdoption(p, '30d'), 31231 + 1976, 'raw clone count leaked into adoption'); +}); + +test('the cloner term is a 14-day snapshot, identical under every period', () => { + const p = hmaShaped(); + // Only the install term (npm) re-windows; the cloner term stays 69 everywhere. + assert.equal(rowAdoption(p, '24h') - 50, 69, '24h cloner term is the 14d snapshot'); + assert.equal(rowAdoption(p, '7d') - 400, 69, '7d cloner term is the 14d snapshot'); + assert.equal(rowAdoption(p, '30d') - 1976, 69, '30d cloner term is the 14d snapshot'); + assert.equal(rowAdoption(p, 'all') - 27731, 69, 'all-time cloner term is the 14d snapshot'); + assert.equal(rowAdoption(p, 'custom') - 12, 69, 'custom cloner term is the 14d snapshot'); +}); + +test('a pure re-clone loop (huge clones, tiny deduped cloners) adds ~nothing', () => { + // 100k raw clones but the same single actor -> deduped uniques ~1. Adoption must + // not move with the raw count, and must NOT scale with active days either. + const looped = { github: { clones30d: 100000, clones: 500000, cloneUniques: 1 }, npm: {}, pypi: {}, docker: {}, hf: null }; + assert.equal(rowAdoption(looped, '30d'), 1, 'a re-clone loop must not inflate adoption'); + assert.equal(rowAdoption(looped, 'all'), 1, 'nor over all-time'); +}); + +test('sumAdoption is cloners + installs and ignores raw clone count entirely', () => { + // It has no `clones` input by construction — the only clone term is cloneUniques. + assert.equal(sumAdoption({ cloneUniques: 10, npm: 5, pypi: 3, docker: 2, hf: 1 }), 21); + assert.equal(sumAdoption({}), 0); + assert.equal(sumAdoption({ npm: 7 }), 7); +}); + +test('docker and HF only count under all-time (unchanged behavior)', () => { + const p = { github: { cloneUniques: 5 }, npm: {}, pypi: {}, docker: { totalPulls: 400 }, hf: { downloadsAllTime: 100, downloads30d: 10 } }; + assert.equal(rowAdoption(p, '30d'), 5 + 10, '30d includes HF 30d but not docker'); + assert.equal(rowAdoption(p, 'all'), 5 + 400 + 100, 'all includes docker pulls + HF all-time'); +}); + +test('periodPick maps windows to the right field', () => { + const o = { a: 1, k30: 2, k7: 3, k24: 4, kc: 5 }; + assert.equal(periodPick(o, 'a', 'k30', 'k7', 'k24', '24h', 'kc'), 4); + assert.equal(periodPick(o, 'a', 'k30', 'k7', 'k24', '7d', 'kc'), 3); + assert.equal(periodPick(o, 'a', 'k30', 'k7', 'k24', '30d', 'kc'), 2); + assert.equal(periodPick(o, 'a', 'k30', 'k7', 'k24', 'custom', 'kc'), 5); + assert.equal(periodPick(o, 'a', 'k30', 'k7', 'k24', 'all', 'kc'), 1); + assert.equal(periodPick(null, 'a', 'k30', 'k7', 'k24', '30d', 'kc'), 0, 'missing object -> 0'); +}); + +test('hfPeriod only reports all-time and 30d', () => { + const hf = { downloadsAllTime: 100, downloads30d: 10 }; + assert.equal(hfPeriod(hf, 'all'), 100); + assert.equal(hfPeriod(hf, '30d'), 10); + assert.equal(hfPeriod(hf, '7d'), null); + assert.equal(hfPeriod(null, 'all'), null); +}); diff --git a/test/repos.test.js b/test/repos.test.js index ca48338c..aa658446 100644 --- a/test/repos.test.js +++ b/test/repos.test.js @@ -7,13 +7,14 @@ const { } = require('../lib/repos'); // Overview-shaped repoStat factory — only the fields the merge reads. -function row(owner, repo, canonical, { views = 0, clones = 0, stars = 0, forks = 0, archived = false } = {}) { +function row(owner, repo, canonical, { views = 0, clones = 0, uniqueCloners = 0, stars = 0, forks = 0, archived = false } = {}) { return { name: `${owner}/${repo}`, owner, repo, canonicalFullName: canonical || `${owner}/${repo}`, totalViews: views, views24h: 0, views7d: 0, views30d: 0, customViews: 0, totalClones: clones, clones24h: 0, clones7d: 0, clones30d: 0, customClones: 0, - recentUniqueVisitors: 0, recentUniqueCloners: 0, + // 14-day deduplicated distinct cloners — a snapshot, taken from the canonical row. + recentUniqueVisitors: 0, recentUniqueCloners: uniqueCloners, stars, starsGrowth24h: 0, starsGrowth7d: 0, starsGrowth30d: 0, starsGrowthAll: 0, starsGrowthCustom: 0, forks, archived, }; @@ -30,15 +31,16 @@ test('canonicalNameOf / ownNameOf read both row shapes', () => { test('mergeByCanonical: transfer twin — traffic sums, stars from canonical (live) row', () => { // Stale opena2a-org row + live opena2a-standards row for the same spec. const merged = mergeByCanonical([ - row('opena2a-org', 'agent-trust-protocol', 'opena2a-standards/agent-trust-protocol', { views: 33, clones: 114, stars: 2 }), - row('opena2a-standards', 'agent-trust-protocol', 'opena2a-standards/agent-trust-protocol', { views: 35, clones: 69, stars: 2 }), + row('opena2a-org', 'agent-trust-protocol', 'opena2a-standards/agent-trust-protocol', { views: 33, clones: 114, uniqueCloners: 8, stars: 2 }), + row('opena2a-standards', 'agent-trust-protocol', 'opena2a-standards/agent-trust-protocol', { views: 35, clones: 69, uniqueCloners: 5, stars: 2 }), ]); assert.strictEqual(merged.length, 1, 'twins collapse to one logical repo'); const m = merged[0]; assert.strictEqual(m.name, 'opena2a-standards/agent-trust-protocol', 'reported under canonical path'); assert.strictEqual(m.owner, 'opena2a-standards'); assert.strictEqual(m.totalViews, 68, 'views summed across twins'); - assert.strictEqual(m.totalClones, 183, 'clones summed across twins'); + assert.strictEqual(m.totalClones, 183, 'raw clones summed across twins'); + assert.strictEqual(m.recentUniqueCloners, 5, 'deduped unique cloners taken from canonical row (NOT summed to 13 — a 14-day dedup cannot be added across twins)'); assert.strictEqual(m.stars, 2, 'stars NOT summed — taken from canonical row (would be 4 if summed)'); assert.strictEqual(m.aliasCount, 1); }); @@ -82,10 +84,12 @@ test('canonicalRepoTotals: dedups twins against a real DB schema', () => { CREATE TABLE repositories (id INTEGER PRIMARY KEY, owner TEXT, repo TEXT, full_name TEXT UNIQUE, canonical_full_name TEXT, archived INTEGER DEFAULT 0); CREATE TABLE stargazers (id INTEGER PRIMARY KEY, repo_id INTEGER, date TEXT, total_stars INTEGER); CREATE TABLE forks (id INTEGER PRIMARY KEY, repo_id INTEGER, date TEXT, total_forks INTEGER); + CREATE TABLE traffic_summary (id INTEGER PRIMARY KEY, repo_id INTEGER, date TEXT, views_count INTEGER, views_uniques INTEGER, clones_count INTEGER, clones_uniques INTEGER); `); const addRepo = db.prepare('INSERT INTO repositories (id, owner, repo, full_name, canonical_full_name, archived) VALUES (?,?,?,?,?,?)'); const addStar = db.prepare('INSERT INTO stargazers (repo_id, date, total_stars) VALUES (?,?,?)'); const addFork = db.prepare('INSERT INTO forks (repo_id, date, total_forks) VALUES (?,?,?)'); + const addSummary = db.prepare('INSERT INTO traffic_summary (repo_id, date, views_count, views_uniques, clones_count, clones_uniques) VALUES (?,?,0,0,0,?)'); // 1: live repo (3 stars). 2: stale transfer twin of 3. 3: canonical of the twin (2 stars). // 4: archived real repo, last snapshot is older (2 stars) — must still count. @@ -99,9 +103,16 @@ test('canonicalRepoTotals: dedups twins against a real DB schema', () => { addStar.run(3, '2026-06-18', 2); addFork.run(3, '2026-06-18', 0); // canonical addStar.run(4, '2026-05-24', 2); addFork.run(4, '2026-05-24', 5); // archived, older date + // Deduped unique cloners: canonical row of the twin wins (5), stale twin's 9 dropped. + addSummary.run(1, '2026-06-18', 30); // live + addSummary.run(2, '2026-05-24', 9); // stale twin — MUST be dropped, not summed + addSummary.run(3, '2026-06-18', 5); // canonical + addSummary.run(4, '2026-05-24', 2); // archived + const t = canonicalRepoTotals(db); db.close(); assert.strictEqual(t.repoCount, 3, 'twin collapses: hackmyagent, spec, killchain'); assert.strictEqual(t.stars, 34, '30 + 2 (canonical spec, not the 9 twin) + 2 (archived)'); assert.strictEqual(t.forks, 7, '2 + 0 + 5'); + assert.strictEqual(t.uniqueCloners, 37, '30 + 5 (canonical spec, NOT the 9 stale twin) + 2 (archived)'); }); diff --git a/test/standards.test.js b/test/standards.test.js index c6c74089..5d5f04b4 100644 --- a/test/standards.test.js +++ b/test/standards.test.js @@ -3,11 +3,13 @@ const assert = require('node:assert'); const { groupStandards, STANDARDS_ORG } = require('../lib/standards'); // Minimal repoStat row factory — only the fields groupStandards reads. -function repo(owner, name, { views = 0, clones = 0, stars = 0, forks = 0, views7d = 0, clones7d = 0, starsGrowthAll = 0 } = {}) { +function repo(owner, name, { views = 0, clones = 0, uniqueCloners = 0, stars = 0, forks = 0, views7d = 0, clones7d = 0, starsGrowthAll = 0 } = {}) { return { owner, repo: name, name: `${owner}/${name}`, totalViews: views, views24h: 0, views7d, views30d: 0, customViews: 0, totalClones: clones, clones24h: 0, clones7d, clones30d: 0, customClones: 0, + // 14-day deduplicated distinct cloners — a snapshot, from the canonical row. + recentUniqueVisitors: 0, recentUniqueCloners: uniqueCloners, stars, starsGrowth24h: 0, starsGrowth7d: 0, starsGrowth30d: 0, starsGrowthAll, starsGrowthCustom: 0, forks, }; @@ -21,16 +23,17 @@ const families = [ test('transferred repo merges stale opena2a-org row: traffic sums, stars from canonical', () => { const rows = [ // Stale pre-transfer row left behind under opena2a-org. - repo('opena2a-org', 'spec-a', { views: 30, clones: 100, stars: 5, forks: 1 }), + repo('opena2a-org', 'spec-a', { views: 30, clones: 100, uniqueCloners: 9, stars: 5, forks: 1 }), // Live row under the standards org (current stars/forks). - repo(STANDARDS_ORG, 'spec-a', { views: 12, clones: 8, stars: 7, forks: 2 }), + repo(STANDARDS_ORG, 'spec-a', { views: 12, clones: 8, uniqueCloners: 4, stars: 7, forks: 2 }), ]; const { standards } = groupStandards(rows, families); const famA = standards.find(f => f.name === 'Fam A'); const specA = famA.repos.find(r => r.repo === 'spec-a'); assert.strictEqual(specA.github.views, 42, 'views summed across both rows'); - assert.strictEqual(specA.github.clones, 108, 'clones summed across both rows'); + assert.strictEqual(specA.github.clones, 108, 'raw clones summed across both rows'); + assert.strictEqual(specA.github.cloneUniques, 4, 'deduped unique cloners from canonical standards-org row (NOT summed to 13 — a 14-day dedup cannot be added across twins)'); assert.strictEqual(specA.github.stars, 7, 'stars taken from canonical standards-org row, not summed'); assert.strictEqual(specA.github.forks, 2, 'forks taken from canonical row'); assert.strictEqual(specA.name, `${STANDARDS_ORG}/spec-a`, 'canonical name wins');