diff --git a/docs/docs/languages/nim-wasm.mdx b/docs/docs/languages/nim-wasm.mdx index 84940c6a0..36791cb02 100644 --- a/docs/docs/languages/nim-wasm.mdx +++ b/docs/docs/languages/nim-wasm.mdx @@ -62,7 +62,7 @@ The [Nim compiler](https://nim-lang.org/) (Nim's `c` backend) compiled to WebAss ### Version -Nim `v2.2.4`, Clang `v22.1.8`. +Nim `v2.2.12`, Clang `v22.1.8`. ## Custom Settings @@ -72,6 +72,8 @@ The compiler can be configured using [custom settings](../advanced/custom-settin | --- | --- | --- | | `compileArgs` | `string[]` | Extra Nim flags, e.g. `["--define:release2"]`. | | `args` | `string[]` | Command-line arguments passed to the program. | +| `packages` | `boolean` | Import external packages from the bundled registry. Defaults to `true`. | +| `libraries` | `string[] \| object` | Public GitHub links or inline library entries added to the bundled registry. See [Adding a library](#adding-a-library). | ```json title="Custom Settings" { @@ -82,6 +84,73 @@ The compiler can be configured using [custom settings](../advanced/custom-settin } ``` +## External Packages + +Programs can import one of 41 Nimble packages from a pinned registry. A package is fetched — its source +files, from its Git repository at a fixed commit — only when a program imports it, so the initial +download is unchanged: + +```nim +import jsony + +type User = object + name: string + age: int + +echo """{"name":"Ada","age":36}""".fromJson(User) +echo User(name: "Ada", age: 36).toJson +``` + +This target compiles through Nim's C backend, so the packages that need C memory or `stdin`/`stdout` +are available as well: `compactdict`, `mustache`, `zippy`, `msgpack4nim`, `nimsvg`, `cligen`, `argparse`, +`docopt` and `nimcrypto`. `results` works here too, as it does on the JavaScript target. The +JavaScript-only packages (`karax`, `nimquery`, `strunicode`) are not. The rest of the 41-package registry +— JSON/YAML/TOML, parsers, text and Unicode (`stew` included), maths, data structures, iterator and +functional helpers (`zero_functional` included), colour, serialization/compression, templating and timing — +works here too. Other Nimble packages are not available, and packages that need a C library, sockets, +threads or a subprocess cannot run in the browser. + +### Adding a library + +The registry above is fixed. `libraries` adds entries to it from [custom settings](../advanced/custom-settings.mdx), without rebuilding anything: + +```json title="Custom Settings" +{ + "nim-wasm": { + "libraries": ["https://github.com/nim-lang/checksums"] + } +} +``` + +It can then be used in a program: + +```nim +import checksums/sha1 + +echo $secureHash("Hello World") +``` + +Each array item is a public GitHub repository link (optionally `/tree/`), or an object with `url` and optional `name`, `ref`, `modules`, `requires` and `backends`. Sources that are not in a public repository can be inlined: + +```json title="Custom Settings" +{ + "nim-wasm": { + "libraries": { + "mylib": { + "modules": ["mylib"], + "files": { + "mylib.nim": "proc hi*(): string = \"hi\"\n" + } + } + } + } +} +``` + +A library is resolved in the worker when the program runs, and the bundled registry stays available. User libraries are not hash-pinned: they are whatever the repository or the custom settings supply. Only public GitHub repositories are fetched; for private sources, use the package's `libraryFromFiles` API. + +Set `"packages": false` to turn package imports off. Adding an entry to `libraries` enables the package loader even when `packages` is `false`, and brings the bundled registry with it. + ## Live Reload By default, new code changes are sent to the result page without a full page reload, avoiding the need to re-download the compiler. Adding the code comment `# __livecodes_reload__` forces a full page reload. diff --git a/docs/docs/languages/nim.mdx b/docs/docs/languages/nim.mdx index 627d82ea6..e1f50b7ab 100644 --- a/docs/docs/languages/nim.mdx +++ b/docs/docs/languages/nim.mdx @@ -46,6 +46,78 @@ The `c` and `cpp` targets compile and run in a worker, like [C (Wasm)](./c-wasm. With either target, the **compiled** code view shows what Nim emitted: the JavaScript program, or the C/C++ that was handed to Clang. +### External packages + +The `nim` language can import a set of Nimble packages from a pinned registry. Nothing is downloaded +until a program imports one, and then only that package's source files are fetched (from its Git +repository at a fixed commit), so a program that uses none costs nothing extra: + +```nim +import jsony + +type User = object + name: string + age: int + +echo """{"name":"Ada","age":36}""".fromJson(User) +echo User(name: "Ada", age: 36).toJson +``` + +The 41-package registry covers JSON, YAML and TOML (`jsony`, `yaml`, `parsetoml`), parsers (`npeg`), text +and Unicode (`regex`, `unicodedb`, `graphemes`, `normalize`, `strunicode`, `stew`), maths (`bigints`, +`vmath`), data structures (`rbtree`, `binaryheap`, `compactdict`), iterator, functional and macro helpers +(`itertools`, `iterutils`, `optionsutils`, `patty`, `with`, `zero_functional`, `unpack`, `fusion`, +`macroutils`, `ast_pattern_matching`), colour (`chroma`, `termstyle`), serialization and compression +(`jstin`, `msgpack4nim`, `zippy`, `pnm`, `nimsvg`), command lines (`cligen`, `argparse`, `docopt`), +templating (`mustache`), UI (`karax`, `nimquery`), timing (`timeit`) and cryptography (`nimcrypto`). + +Some packages only work on one backend: `compactdict`, `mustache`, `zippy`, `msgpack4nim`, `nimsvg`, +`cligen`, `argparse`, `docopt` and `nimcrypto` need the C/C++ targets (the `wasm` target compiles through +C, so they work there too), while `karax`, `nimquery` and `strunicode` need the JavaScript target. +`results` works on both. Other Nimble packages are not available, and packages that need a C library, +sockets, threads or a subprocess cannot run in the browser. + +### Adding a library + +The registry above is fixed. `libraries` adds entries to it from [custom settings](../advanced/custom-settings.mdx), without rebuilding anything: + +```json title="Custom Settings" +{ + "nim": { + "libraries": ["https://github.com/nim-lang/checksums"] + } +} +``` + +It can then be used in a program: + +```nim +import checksums/sha1 + +echo $secureHash("Hello World") +``` + +Each array item is a public GitHub repository link (optionally `/tree/`), or an object with `url` and optional `name`, `ref`, `modules`, `requires` and `backends`. Sources that are not in a public repository can be inlined: + +```json title="Custom Settings" +{ + "nim": { + "libraries": { + "mylib": { + "modules": ["mylib"], + "files": { + "mylib.nim": "proc hi*(): string = \"hi\"\n" + } + } + } + } +} +``` + +A library is resolved when the program runs, and the bundled registry stays available. User libraries are not hash-pinned: they are whatever the repository or the custom settings supply. Only public GitHub repositories are fetched; for private sources, use the package's `libraryFromFiles` API. + +Set `"packages": false` in custom settings to turn package imports off. Adding an entry to `libraries` enables the package loader even when `packages` is `false`, and brings the bundled registry with it. + ### Communication with the result page With the default `js` backend, Nim emits one self-sufficient program that runs in the result page. It has a document and a console, so `echo` writes to the console, and the browser modules that ship with the compiler (`dom`, `jsffi`, `jsconsole`, `jscore`, `jsre` and `asyncjs`) are on its search path: @@ -83,7 +155,7 @@ The [Nim compiler](https://nim-lang.org/) (Nim's `js`, `c` or `cpp` backend) com ### Version -Nim `v2.2.4`. Clang `v22.1.8` for the `c` and `cpp` targets. +Nim `v2.2.12`. Clang `v22.1.8` for the `c` and `cpp` targets. ## Custom Settings @@ -92,6 +164,8 @@ The compiler can be configured using [custom settings](../advanced/custom-settin | Property | Type | Description | | --- | --- | --- | | `target` | `"js" \| "c" \| "cpp"` | The Nim backend. Defaults to `js`. | +| `packages` | `boolean` | Import external packages from the bundled registry. Defaults to `true`. | +| `libraries` | `string[] \| object` | Public GitHub links or inline library entries added to the bundled registry. See [Adding a library](#adding-a-library). | ```json title="Custom Settings" { diff --git a/src/livecodes/languages/nim/__tests__/lang-nim.spec.ts b/src/livecodes/languages/nim/__tests__/lang-nim.spec.ts index 285b89d0d..5a9dadd45 100644 --- a/src/livecodes/languages/nim/__tests__/lang-nim.spec.ts +++ b/src/livecodes/languages/nim/__tests__/lang-nim.spec.ts @@ -56,6 +56,9 @@ const createCompilerMock = () => { run: async () => ({ compiledCode: `compiled-${options.target}` }), }); }, + resolveLibraries: async () => ({ + demo: { name: 'demo', modules: ['demo'], files: { 'demo.nim': '' } }, + }), }; return calls; }; @@ -77,7 +80,7 @@ describe('nim compiler factory', () => { }); expect(calls).toHaveLength(1); - expect(calls[0]).toMatchObject({ target: 'c', emit: 'single' }); + expect(calls[0]).toMatchObject({ target: 'c', emit: 'single', packages: true }); expect(result).toEqual({ code: 'compiled-c', info: {} }); }); @@ -101,4 +104,37 @@ describe('nim compiler factory', () => { await compile('c', { config: configWith({ nim: { target: 'c' } }) }); expect(calls.map((call) => call.target)).toEqual(['cpp', 'c']); }); + + test('resolves custom libraries and caches the compiler by them', async () => { + const calls = createCompilerMock(); + const compile = await compilerFactory(); + const config = configWith({ + nim: { target: 'js', libraries: ['https://github.com/acme/demo'] }, + }); + + await compile('a', { config }); + expect(calls).toHaveLength(1); + expect(calls[0].packages).toMatchObject({ + demo: { name: 'demo', modules: ['demo'], files: { 'demo.nim': '' } }, + }); + + await compile('b', { config }); + expect(calls).toHaveLength(1); + }); + test('packages are on by default and can be turned off', async () => { + const calls = createCompilerMock(); + const compile = await compilerFactory(); + const js = { config: configWith({ nim: { target: 'js' } }) }; + const jsWithoutPackages = { + config: configWith({ nim: { target: 'js', packages: false } }), + }; + + await compile('a', js); + await compile('b', jsWithoutPackages); + await compile('c', jsWithoutPackages); + + // A different `packages` setting is a different compiler, and the same one is reused. + expect(calls).toHaveLength(2); + expect(calls.map((call) => call.packages)).toEqual([true, false]); + }); }); diff --git a/src/livecodes/languages/nim/lang-nim-wasm-script.ts b/src/livecodes/languages/nim/lang-nim-wasm-script.ts index 89bd4bbb8..52b5e891c 100644 --- a/src/livecodes/languages/nim/lang-nim-wasm-script.ts +++ b/src/livecodes/languages/nim/lang-nim-wasm-script.ts @@ -16,6 +16,8 @@ const SCRIPT_TYPE = 'text/nim-wasm'; interface NimWasmSettings { compileArgs?: string[]; args?: string[]; + packages?: boolean; + libraries?: unknown; } interface RunResult { @@ -60,12 +62,24 @@ const getToolchain = () => { return toolchain; }; -const getCompiler = (compileArgs = []) => { - const key = JSON.stringify(compileArgs); +const libraryCache = new Map(); +const getCompiler = async (options = {}) => { + const compileArgs = options.compileArgs || []; + const requestedPackages = options.packages !== false; + const libraries = options.libraries; + const key = JSON.stringify({ + compileArgs, + packages: requestedPackages, + libraries: libraries ?? null, + }); if (compiler && compilerArgs === key) return compiler; compilerArgs = key; const previous = compiler; const created = (async () => { + const entries = libraries + ? await self.nimWasm.resolveLibraries(libraries, { cache: libraryCache }) + : {}; + const packages = Object.keys(entries).length ? entries : requestedPackages; // Compile through the Clang runtime this page already uses, rather than the second copy the // Nim IIFE carries inside it, so only one runtime is ever loaded. const built = await getToolchain(); @@ -74,9 +88,11 @@ const getCompiler = (compileArgs = []) => { baseUrl: ${JSON.stringify(nimWasmBaseUrl + 'assets/nim/')}, toolchain: built, compileArgs, + packages, }); - // compileArgs is fixed when a compiler is created, so a change builds a new one and - // releases the old; the shared toolchain belongs to this page and is not released. + // compileArgs, packages and libraries are fixed when a compiler is created, so a change + // builds a new one and releases the old; the shared toolchain belongs to this page and is + // not released. if (previous) previous.then((old) => old.dispose()).catch(() => undefined); return next; })(); @@ -93,11 +109,7 @@ const getCompiler = (compileArgs = []) => { addEventListener('message', async (event) => { const { id, code, input, options } = event.data; try { - const result = await (await getCompiler(options && options.compileArgs)).run( - code, - input, - options, - ); + const result = await (await getCompiler(options)).run(code, input, options); postMessage({ id, result: { diff --git a/src/livecodes/languages/nim/lang-nim.ts b/src/livecodes/languages/nim/lang-nim.ts index 1b99cdc80..af01edd66 100644 --- a/src/livecodes/languages/nim/lang-nim.ts +++ b/src/livecodes/languages/nim/lang-nim.ts @@ -4,36 +4,61 @@ import { getLanguageCustomSettings } from '../utils'; // Nim's JavaScript backend, for pages that want a document: the compiler emits one self-sufficient // file, so it is compiled with `execute: false` and the result is run in the result page, the same -// shape as the TypeScript compiler's factory. +// shape as the TypeScript compiler's factory. External packages are fetched on first import. export const nim: LanguageSpecs = { name: 'nim', title: 'Nim', compiler: { url: nimWasmBaseUrl + 'dist/nim-wasm.global.js', factory: () => { - const compilers: Record<'js' | 'c' | 'cpp', Promise | null> = { - js: null, - c: null, - cpp: null, - }; - const getCompiler = (target: 'js' | 'c' | 'cpp' = 'js') => { - if (!compilers[target]) { - const created: Promise = (self as any).nimWasm.createCompiler({ - target, - emit: 'single', - baseUrl: nimWasmBaseUrl + 'assets/nim/', - }); + const compilers = new Map>(); + const libraryCache = new Map>(); + const getCompiler = async ( + target: 'js' | 'c' | 'cpp' = 'js', + packages = true, + libraries?: unknown, + ) => { + const key = `${target}:${packages}:${JSON.stringify(libraries ?? null)}`; + let compiler = compilers.get(key); + if (!compiler) { + const created: Promise = (async () => { + const entries = libraries + ? await (self as any).nimWasm.resolveLibraries(libraries, { cache: libraryCache }) + : {}; + return (self as any).nimWasm.createCompiler({ + target, + emit: 'single', + baseUrl: nimWasmBaseUrl + 'assets/nim/', + // An object is merged with the bundled registry; a user library therefore enables the + // package loader even if `packages` was false. + packages: Object.keys(entries).length ? entries : packages, + }); + })(); // do not keep a failed boot (e.g. a transient network error) created.catch(() => { - compilers[target] = null; + if (compilers.get(key) === created) compilers.delete(key); }); - compilers[target] = created; + compilers.set(key, created); + compiler = created; } - return compilers[target]; + return compiler; }; return async (code: string, { config }) => { - const target = getLanguageCustomSettings('nim', config)?.target || 'js'; - const result = await (await getCompiler(target)).run(code, '', { execute: false }); + const settings = getLanguageCustomSettings('nim', config) ?? {}; + const target = settings.target || 'js'; + const packages = settings.packages !== false; + let compiler: any; + try { + compiler = await getCompiler(target, packages, settings.libraries); + } catch (error) { + return { + code: '', + info: { + errors: [`Could not load Nim libraries: ${(error as Error)?.message ?? error}`], + }, + }; + } + const result = await compiler.run(code, '', { execute: false }); if (!result.compiledCode) { return { code: '', info: { errors: result.errors || ['Nim produced no output.'] } }; } diff --git a/src/livecodes/vendors.ts b/src/livecodes/vendors.ts index 2942062e0..3d91d8852 100644 --- a/src/livecodes/vendors.ts +++ b/src/livecodes/vendors.ts @@ -385,7 +385,7 @@ export const monacoVolarUrl = /* @__PURE__ */ getUrl( export const mustacheUrl = /* @__PURE__ */ getUrl('mustache@4.2.0/mustache.js'); -export const nimWasmBaseUrl = /* @__PURE__ */ getUrl('@live-codes/nim-wasm@0.5.0/'); +export const nimWasmBaseUrl = /* @__PURE__ */ getUrl('@live-codes/nim-wasm@0.6.0/'); export const ninjaKeysUrl = /* @__PURE__ */ getUrl('@hatemhosny/ninja-keys@1.14.0/bundle/index.js');