diff --git a/src/news-sources.mjs b/src/news-sources.mjs index 36e1b86..7ec3370 100644 --- a/src/news-sources.mjs +++ b/src/news-sources.mjs @@ -20,6 +20,8 @@ // holds feeds with stable well-known URLs and defers everything else to the // published lists, where the list is somebody else's to maintain. +import fs from "node:fs"; + /** * Bing News query feed — the only search feed left. * @@ -86,6 +88,53 @@ export function isDeadEndLink(url) { } } +/** The vendored copy of what profullstack.com/feeds.opml serves. */ +const PROFULLSTACK_OPML = new URL("./profullstack-feeds.opml", import.meta.url); + +/** slugify() from news.mjs, kept in step by a test rather than imported. */ +function slug(label) { + return String(label ?? "") + .toLowerCase() + .replace(/['’]/g, "") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 32); +} + +/** + * The profullstack blogs, read from the vendored OPML rather than typed out. + * + * Read synchronously and at module load because defaultFeeds() is synchronous — + * a fresh install must not wait on a network call, or on a promise, to show + * anything at all. The file ships with the package (`files` includes `src`), so + * it is there in both the npm and the install.sh channel. + * + * Parsed here with a small matcher instead of news.mjs's parseOpml, because + * news.mjs imports this module and taking the import back the other way makes a + * cycle. The matcher can afford to be small: this is our own file, flat, and a + * test asserts the two agree on every feed it contains. + * + * A missing or unreadable file degrades to no profullstack defaults rather than + * throwing, which would take `/news` down entirely over a packaging mistake. + */ +function profullstackFeeds() { + let xml; + try { xml = fs.readFileSync(PROFULLSTACK_OPML, "utf8"); } + catch { return []; } + + const feeds = []; + const seen = new Set(); + for (const tag of xml.match(/]*>/gi) ?? []) { + const attr = (name) => (new RegExp(`\\b${name}="([^"]*)"`, "i").exec(tag) ?? [])[1] ?? ""; + const url = attr("xmlUrl"); + if (!/^https?:\/\//i.test(url) || seen.has(url)) continue; + seen.add(url); + const title = attr("title") || attr("text") || url; + feeds.push({ name: slug(title), title, url, site: attr("htmlUrl"), category: "profullstack" }); + } + return feeds; +} + /** * The feeds a fresh install reads. * @@ -120,25 +169,18 @@ export const DEFAULT_FEEDS = [ { name: "npr-politics", title: "NPR — Politics", url: "https://feeds.npr.org/1014/rss.xml", site: "https://www.npr.org", category: "politics" }, - // The profullstack blogs, read out of the box. This is the one published list - // small enough to be a default: profullstack.com/feeds.opml is 14 feeds, where - // smallweb is 33,000 and could only ever be searched. Kept in step with that - // file by hand rather than fetched, because defaultFeeds() is synchronous and - // a fresh install must not wait on a network call to show anything at all. - { name: "bittorrented-blog", title: "BitTorrented Blog", url: "https://bittorrented.com/blog/rss.xml", site: "https://bittorrented.com/blog", category: "profullstack" }, - { name: "bl0ggers-blog", title: "bl0ggers Blog", url: "https://bl0ggers.com/blog/rss.xml", site: "https://bl0ggers.com/blog", category: "profullstack" }, - { name: "c0mpute-blog", title: "c0mpute Blog", url: "https://c0mpute.com/blog/rss.xml", site: "https://c0mpute.com/blog", category: "profullstack" }, - { name: "c0upons-blog", title: "c0upons Blog", url: "https://c0upons.com/blog/rss.xml", site: "https://c0upons.com/blog", category: "profullstack" }, - { name: "coinpay-blog", title: "CoinPay Blog", url: "https://coinpayportal.com/blog/rss.xml", site: "https://coinpayportal.com/blog", category: "profullstack" }, - { name: "crawlproof-blog", title: "CrawlProof Blog", url: "https://crawlproof.com/blog/rss.xml", site: "https://crawlproof.com/blog", category: "profullstack" }, - { name: "d0rz-blog", title: "d0rz Blog", url: "https://d0rz.com/blog/rss.xml", site: "https://d0rz.com/blog", category: "profullstack" }, - { name: "logicsrc-blog", title: "LogicSRC Blog", url: "https://logicsrc.com/blog/rss.xml", site: "https://logicsrc.com/blog", category: "profullstack" }, - { name: "pairux-blog", title: "PairUX Blog", url: "https://pairux.com/blog/rss.xml", site: "https://pairux.com/blog", category: "profullstack" }, - { name: "qryptchat-blog", title: "QryptChat Blog", url: "https://qrypt.chat/blog/rss.xml", site: "https://qrypt.chat/blog", category: "profullstack" }, - { name: "saasrow-blog", title: "SaaSRow Blog", url: "https://www.saasrow.com/blog/rss.xml", site: "https://www.saasrow.com/blog", category: "profullstack" }, - { name: "sh1pt-blog", title: "sh1pt Blog", url: "https://sh1pt.com/blog/rss.xml", site: "https://sh1pt.com/blog", category: "profullstack" }, - { name: "threatcrush-blog", title: "ThreatCrush Blog", url: "https://threatcrush.com/blog/rss.xml", site: "https://threatcrush.com/blog", category: "profullstack" }, - { name: "ugig-blog", title: "ugig Blog", url: "https://ugig.net/blog/rss.xml", site: "https://ugig.net/blog", category: "profullstack" }, + // The profullstack blogs, read out of the box. Not typed out here: they are + // parsed from src/profullstack-feeds.opml, which is a copy of what + // profullstack.com/feeds.opml actually serves. Two hand-maintained lists of + // the same fourteen blogs is one more than can be kept in step, and the copy + // that would go stale is this one — nobody editing the published OPML has a + // reason to think about moshcode. Refresh it with: + // + // curl -sL https://profullstack.com/feeds.opml -o src/profullstack-feeds.opml + // + // test/profullstack-feeds.test.mjs checks that against the live file when + // MOSHCODE_CHECK_FEED_DRIFT=1 is set. + ...profullstackFeeds(), ]; /** diff --git a/src/profullstack-feeds.opml b/src/profullstack-feeds.opml new file mode 100644 index 0000000..e4bf191 --- /dev/null +++ b/src/profullstack-feeds.opml @@ -0,0 +1,25 @@ + + + + Profullstack Blogs + Profullstack + https://profullstack.com + Thu, 13 Aug 2026 00:00:00 +0000 + + + + + + + + + + + + + + + + + + diff --git a/src/settings-sync.mjs b/src/settings-sync.mjs index 40344b6..f018b80 100644 --- a/src/settings-sync.mjs +++ b/src/settings-sync.mjs @@ -64,6 +64,16 @@ export const SYNCED_FILES = [ // and read by nothing here — moshcode's interest in it begins and ends with // moving it, and a file this does not parse cannot be broken by this. { path: "feeds.opml", json: false, label: "rss feeds" }, + // `/news` and `/rss` keep their subscriptions here — see opmlFile() in + // news.mjs. A near-identical name sat in this list above it for a while and + // the two were easy to mistake for each other, so: feeds.opml belongs to + // tcfeed, news.opml belongs to this. Carrying only the first meant the feeds + // you actually subscribed to were the one thing `/save` left behind. + // + // Deliberately no cap change. A subscription list past MAX_FILE_BYTES is + // reported as skipped rather than failing the snapshot, which is the right + // answer for a list that got big by importing somebody else's. + { path: "news.opml", json: false, label: "news subscriptions" }, ]; /** diff --git a/test/profullstack-feeds.test.mjs b/test/profullstack-feeds.test.mjs new file mode 100644 index 0000000..719ddce --- /dev/null +++ b/test/profullstack-feeds.test.mjs @@ -0,0 +1,90 @@ +// The vendored profullstack feed list, against the published one. +// +// src/profullstack-feeds.opml is a copy of what profullstack.com/feeds.opml +// serves. It is a copy on purpose: defaultFeeds() is synchronous, so a fresh +// install cannot fetch the list before it shows anything. The cost of that +// choice is drift, and the point of this file is to make drift loud. +// +// Everything that can be checked without a network runs always — that the file +// is there, that it parses, that the small matcher in news-sources.mjs agrees +// with the real parseOpml, and that its private slug() still matches +// slugify(). Those are the failures a refactor actually causes. +// +// The one check that needs the network — vendored copy against the live URL — +// is opt-in, because a suite that fails when profullstack.com is briefly down +// is a suite people learn to ignore. Run it deliberately: +// +// MOSHCODE_CHECK_FEED_DRIFT=1 node --test test/profullstack-feeds.test.mjs +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { DEFAULT_FEEDS } from "../src/news-sources.mjs"; +import { parseOpml, slugify } from "../src/news.mjs"; + +const OPML = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "src", "profullstack-feeds.opml"); +const LIVE = "https://profullstack.com/feeds.opml"; + +const vendored = fs.readFileSync(OPML, "utf8"); +const mine = DEFAULT_FEEDS.filter((f) => f.category === "profullstack"); + +test("the vendored OPML ships and is read", () => { + // `files` in package.json includes `src`, so this travels with the package. + // If it ever stops, profullstackFeeds() degrades to [] rather than throwing — + // which is the right runtime behaviour and exactly why it needs asserting + // here instead: silence is the failure mode. + assert.ok(vendored.includes(" 0, "no profullstack feeds in the defaults"); +}); + +test("every feed in the file is a default, and nothing else is", () => { + // parseOpml is the real parser; news-sources.mjs cannot import it without a + // cycle, so it has a small matcher of its own. This is the assertion that the + // shortcut did not change the answer. + const parsed = parseOpml(vendored); + assert.deepEqual( + mine.map((f) => f.url).sort(), + parsed.map((f) => f.url).sort(), + ); + assert.deepEqual( + mine.map((f) => f.title).sort(), + parsed.map((f) => f.title).sort(), + ); +}); + +test("the private slug() still agrees with slugify()", () => { + // news-sources.mjs copies slugify() rather than importing it, for the same + // cycle reason. A feed named differently by the two would be reachable as + // `--feed ` under one name and listed under another. + for (const feed of mine) assert.equal(feed.name, slugify(feed.title), feed.title); +}); + +test("the feeds are usable — https, unique, and pointed at a site", () => { + const urls = mine.map((f) => f.url); + assert.equal(new Set(urls).size, urls.length, "duplicate feed URL"); + for (const feed of mine) { + assert.match(feed.url, /^https:\/\//, `${feed.title} is not https`); + assert.ok(feed.title.trim(), "a feed with no title"); + assert.match(feed.site, /^https:\/\//, `${feed.title} has no site`); + } +}); + +test("vendored copy matches profullstack.com/feeds.opml", { + skip: process.env.MOSHCODE_CHECK_FEED_DRIFT === "1" + ? false + : "set MOSHCODE_CHECK_FEED_DRIFT=1 to check against the live file", +}, async () => { + const res = await fetch(LIVE); + assert.equal(res.ok, true, `${LIVE} answered ${res.status}`); + const live = parseOpml(await res.text()); + + // Compared by what a reader would act on, not byte for byte: the published + // file carries a dateCreated that changes without any feed changing. + assert.deepEqual( + mine.map((f) => f.url).sort(), + live.map((f) => f.url).sort(), + `vendored list is stale — refresh with:\n curl -sL ${LIVE} -o src/profullstack-feeds.opml`, + ); +}); diff --git a/test/settings-sync.test.mjs b/test/settings-sync.test.mjs index 848428f..0cba6e7 100644 --- a/test/settings-sync.test.mjs +++ b/test/settings-sync.test.mjs @@ -41,7 +41,7 @@ const DIGEST_FIXTURE = { "herd/rules.json": { content: "{}" }, }; -function home({ aliases = null, rules = null, credentials = true, marker = null, feeds = null } = {}) { +function home({ aliases = null, rules = null, credentials = true, marker = null, feeds = null, news = null } = {}) { const dir = mkdtempSync(path.join(tmpdir(), "moshcode-sync-")); const moshcode = path.join(dir, ".moshcode"); fs.mkdirSync(moshcode, { recursive: true }); @@ -51,6 +51,7 @@ function home({ aliases = null, rules = null, credentials = true, marker = null, } if (aliases) fs.writeFileSync(path.join(moshcode, "aliases.json"), aliases); if (feeds) fs.writeFileSync(path.join(moshcode, "feeds.opml"), feeds); + if (news) fs.writeFileSync(path.join(moshcode, "news.opml"), news); if (rules) { fs.mkdirSync(path.join(moshcode, "herd"), { recursive: true }); fs.writeFileSync(path.join(moshcode, "herd", "rules.json"), rules); @@ -275,6 +276,27 @@ test("/save carries the feed list, and does not try to parse it", async () => { assert.equal(sent["feeds.opml"].content, OPML, "OPML travels byte for byte"); }); +test("/save carries the news subscriptions too, not only tcfeed's list", async () => { + // These two files sat one line apart in SYNCED_FILES with nearly the same + // name, and only tcfeed's was in it. `/news` and `/rss` keep their + // subscriptions in news.opml, so the feeds you actually chose were the one + // thing `/save` left behind. + const dir = home({ aliases: "{}", feeds: OPML, news: OPML }); + const fetchImpl = stubFetch([[200, { revision: 1 }]]); + + const code = await saveCommand([], { home: dir, creds: CREDS, fetchImpl, write: lines(), installed: INSTALLED }); + assert.equal(code, 0); + const sent = fetchImpl.calls[0].body.snapshot.files; + assert.equal(sent["news.opml"].content, OPML, "the subscription list did not travel"); + assert.ok(sent["feeds.opml"], "tcfeed's list still travels"); +}); + +test("news.opml is in the allowlist and is not parsed", () => { + const entry = SYNCED_FILES.find((f) => f.path === "news.opml"); + assert.ok(entry, "news.opml is not synced"); + assert.equal(entry.json, false, "OPML must not be JSON-parsed"); +}); + test("a feed list that is not XML still syncs — nothing here reads it", async () => { // The contrast with aliases.json is the point: that one is `json: true` and // a broken one is held back rather than copied to every machine. OPML is