Support Nim libraries

docs/docs/languages/nim-wasm.mdx

@@ -62,7 +62,7 @@ The [Nim compiler](https://nim-lang.org/) (Nim's `c` backend) compiled to WebAss
 
 ### Version
 
-Nim `v2.2.4`, Clang `v22.1.8`.
+Nim `v2.2.12`, Clang `v22.1.8`.
 
 ## Custom Settings
 
@@ -72,6 +72,8 @@ The compiler can be configured using [custom settings](../advanced/custom-settin
 | --- | --- | --- |
 | `compileArgs` | `string[]` | Extra Nim flags, e.g. `["--define:release2"]`. |
 | `args` | `string[]` | Command-line arguments passed to the program. |
+| `packages` | `boolean` | Import external packages from the bundled registry. Defaults to `true`. |
+| `libraries` | `string[] \| object` | Public GitHub links or inline library entries added to the bundled registry. See [Adding a library](#adding-a-library). |
 
 ```json title="Custom Settings"
 {
@@ -82,6 +84,73 @@ The compiler can be configured using [custom settings](../advanced/custom-settin
 }
 ```
 
+## External Packages
+
+Programs can import one of 41 Nimble packages from a pinned registry. A package is fetched — its source
+files, from its Git repository at a fixed commit — only when a program imports it, so the initial
+download is unchanged:
+
+```nim
+import jsony
+
+type User = object
+  name: string
+  age: int
+
+echo """{"name":"Ada","age":36}""".fromJson(User)
+echo User(name: "Ada", age: 36).toJson
+```
+
+This target compiles through Nim's C backend, so the packages that need C memory or `stdin`/`stdout`
+are available as well: `compactdict`, `mustache`, `zippy`, `msgpack4nim`, `nimsvg`, `cligen`, `argparse`,
+`docopt` and `nimcrypto`. `results` works here too, as it does on the JavaScript target. The
+JavaScript-only packages (`karax`, `nimquery`, `strunicode`) are not. The rest of the 41-package registry
+— JSON/YAML/TOML, parsers, text and Unicode (`stew` included), maths, data structures, iterator and
+functional helpers (`zero_functional` included), colour, serialization/compression, templating and timing —
+works here too. Other Nimble packages are not available, and packages that need a C library, sockets,
+threads or a subprocess cannot run in the browser.
+
+### Adding a library
+
+The registry above is fixed. `libraries` adds entries to it from [custom settings](../advanced/custom-settings.mdx), without rebuilding anything:
+
+```json title="Custom Settings"
+{
+  "nim-wasm": {
+    "libraries": ["https://github.com/nim-lang/checksums"]
+  }
+}
+```
+
+It can then be used in a program:
+
+```nim
+import checksums/sha1
+
+echo $secureHash("Hello World")
+```
+
+Each array item is a public GitHub repository link (optionally `/tree/<ref>`), or an object with `url` and optional `name`, `ref`, `modules`, `requires` and `backends`. Sources that are not in a public repository can be inlined:
+
+```json title="Custom Settings"
+{
+  "nim-wasm": {
+    "libraries": {
+      "mylib": {
+        "modules": ["mylib"],
+        "files": {
+          "mylib.nim": "proc hi*(): string = \"hi\"\n"
+        }
+      }
+    }
+  }
+}
+```
+
+A library is resolved in the worker when the program runs, and the bundled registry stays available. User libraries are not hash-pinned: they are whatever the repository or the custom settings supply. Only public GitHub repositories are fetched; for private sources, use the package's `libraryFromFiles` API.
+
+Set `"packages": false` to turn package imports off. Adding an entry to `libraries` enables the package loader even when `packages` is `false`, and brings the bundled registry with it.
+
 ## Live Reload
 
 By default, new code changes are sent to the result page without a full page reload, avoiding the need to re-download the compiler. Adding the code comment `# __livecodes_reload__` forces a full page reload.

docs/docs/languages/nim.mdx

@@ -46,6 +46,78 @@ The `c` and `cpp` targets compile and run in a worker, like [C (Wasm)](./c-wasm.
 
 With either target, the **compiled** code view shows what Nim emitted: the JavaScript program, or the C/C++ that was handed to Clang.
 
+### External packages
+
+The `nim` language can import a set of Nimble packages from a pinned registry. Nothing is downloaded
+until a program imports one, and then only that package's source files are fetched (from its Git
+repository at a fixed commit), so a program that uses none costs nothing extra:
+
+```nim
+import jsony
+
+type User = object
+  name: string
+  age: int
+
+echo """{"name":"Ada","age":36}""".fromJson(User)
+echo User(name: "Ada", age: 36).toJson
+```
+
+The 41-package registry covers JSON, YAML and TOML (`jsony`, `yaml`, `parsetoml`), parsers (`npeg`), text
+and Unicode (`regex`, `unicodedb`, `graphemes`, `normalize`, `strunicode`, `stew`), maths (`bigints`,
+`vmath`), data structures (`rbtree`, `binaryheap`, `compactdict`), iterator, functional and macro helpers
+(`itertools`, `iterutils`, `optionsutils`, `patty`, `with`, `zero_functional`, `unpack`, `fusion`,
+`macroutils`, `ast_pattern_matching`), colour (`chroma`, `termstyle`), serialization and compression
+(`jstin`, `msgpack4nim`, `zippy`, `pnm`, `nimsvg`), command lines (`cligen`, `argparse`, `docopt`),
+templating (`mustache`), UI (`karax`, `nimquery`), timing (`timeit`) and cryptography (`nimcrypto`).
+
+Some packages only work on one backend: `compactdict`, `mustache`, `zippy`, `msgpack4nim`, `nimsvg`,
+`cligen`, `argparse`, `docopt` and `nimcrypto` need the C/C++ targets (the `wasm` target compiles through
+C, so they work there too), while `karax`, `nimquery` and `strunicode` need the JavaScript target.
+`results` works on both. Other Nimble packages are not available, and packages that need a C library,
+sockets, threads or a subprocess cannot run in the browser.
+
+### Adding a library
+
+The registry above is fixed. `libraries` adds entries to it from [custom settings](../advanced/custom-settings.mdx), without rebuilding anything:
+
+```json title="Custom Settings"
+{
+  "nim": {
+    "libraries": ["https://github.com/nim-lang/checksums"]
+  }
+}
+```
+
+It can then be used in a program:
+
+```nim
+import checksums/sha1
+
+echo $secureHash("Hello World")
+```
+
+Each array item is a public GitHub repository link (optionally `/tree/<ref>`), or an object with `url` and optional `name`, `ref`, `modules`, `requires` and `backends`. Sources that are not in a public repository can be inlined:
+
+```json title="Custom Settings"
+{
+  "nim": {
+    "libraries": {
+      "mylib": {
+        "modules": ["mylib"],
+        "files": {
+          "mylib.nim": "proc hi*(): string = \"hi\"\n"
+        }
+      }
+    }
+  }
+}
+```
+
+A library is resolved when the program runs, and the bundled registry stays available. User libraries are not hash-pinned: they are whatever the repository or the custom settings supply. Only public GitHub repositories are fetched; for private sources, use the package's `libraryFromFiles` API.
+
+Set `"packages": false` in custom settings to turn package imports off. Adding an entry to `libraries` enables the package loader even when `packages` is `false`, and brings the bundled registry with it.
+
 ### Communication with the result page
 
 With the default `js` backend, Nim emits one self-sufficient program that runs in the result page. It has a document and a console, so `echo` writes to the console, and the browser modules that ship with the compiler (`dom`, `jsffi`, `jsconsole`, `jscore`, `jsre` and `asyncjs`) are on its search path:
@@ -83,7 +155,7 @@ The [Nim compiler](https://nim-lang.org/) (Nim's `js`, `c` or `cpp` backend) com
 
 ### Version
 
-Nim `v2.2.4`. Clang `v22.1.8` for the `c` and `cpp` targets.
+Nim `v2.2.12`. Clang `v22.1.8` for the `c` and `cpp` targets.
 
 ## Custom Settings
 
@@ -92,6 +164,8 @@ The compiler can be configured using [custom settings](../advanced/custom-settin
 | Property | Type | Description |
 | --- | --- | --- |
 | `target` | `"js" \| "c" \| "cpp"` | The Nim backend. Defaults to `js`. |
+| `packages` | `boolean` | Import external packages from the bundled registry. Defaults to `true`. |
+| `libraries` | `string[] \| object` | Public GitHub links or inline library entries added to the bundled registry. See [Adding a library](#adding-a-library). |
 
 ```json title="Custom Settings"
 {

src/livecodes/languages/nim/__tests__/lang-nim.spec.ts

@@ -56,6 +56,9 @@ const createCompilerMock = () => {
         run: async () => ({ compiledCode: `compiled-${options.target}` }),
       });
     },
+    resolveLibraries: async () => ({
+      demo: { name: 'demo', modules: ['demo'], files: { 'demo.nim': '' } },
+    }),
   };
   return calls;
 };
@@ -77,7 +80,7 @@ describe('nim compiler factory', () => {
     });
 
     expect(calls).toHaveLength(1);
-    expect(calls[0]).toMatchObject({ target: 'c', emit: 'single' });
+    expect(calls[0]).toMatchObject({ target: 'c', emit: 'single', packages: true });
     expect(result).toEqual({ code: 'compiled-c', info: {} });
   });
 
@@ -101,4 +104,37 @@ describe('nim compiler factory', () => {
     await compile('c', { config: configWith({ nim: { target: 'c' } }) });
     expect(calls.map((call) => call.target)).toEqual(['cpp', 'c']);
   });
+
+  test('resolves custom libraries and caches the compiler by them', async () => {
+    const calls = createCompilerMock();
+    const compile = await compilerFactory();
+    const config = configWith({
+      nim: { target: 'js', libraries: ['https://github.com/acme/demo'] },
+    });
+
+    await compile('a', { config });
+    expect(calls).toHaveLength(1);
+    expect(calls[0].packages).toMatchObject({
+      demo: { name: 'demo', modules: ['demo'], files: { 'demo.nim': '' } },
+    });
+
+    await compile('b', { config });
+    expect(calls).toHaveLength(1);
+  });
+  test('packages are on by default and can be turned off', async () => {
+    const calls = createCompilerMock();
+    const compile = await compilerFactory();
+    const js = { config: configWith({ nim: { target: 'js' } }) };
+    const jsWithoutPackages = {
+      config: configWith({ nim: { target: 'js', packages: false } }),
+    };
+
+    await compile('a', js);
+    await compile('b', jsWithoutPackages);
+    await compile('c', jsWithoutPackages);
+
+    // A different `packages` setting is a different compiler, and the same one is reused.
+    expect(calls).toHaveLength(2);
+    expect(calls.map((call) => call.packages)).toEqual([true, false]);
+  });
 });

src/livecodes/languages/nim/lang-nim-wasm-script.ts

@@ -16,6 +16,8 @@ const SCRIPT_TYPE = 'text/nim-wasm';
 interface NimWasmSettings {
   compileArgs?: string[];
   args?: string[];
+  packages?: boolean;
+  libraries?: unknown;
 }
 
 interface RunResult {
@@ -60,12 +62,24 @@ const getToolchain = () => {
   return toolchain;
 };
 
-const getCompiler = (compileArgs = []) => {
-  const key = JSON.stringify(compileArgs);
+const libraryCache = new Map();
+const getCompiler = async (options = {}) => {
+  const compileArgs = options.compileArgs || [];
+  const requestedPackages = options.packages !== false;
+  const libraries = options.libraries;
+  const key = JSON.stringify({
+    compileArgs,
+    packages: requestedPackages,
+    libraries: libraries ?? null,
+  });
   if (compiler && compilerArgs === key) return compiler;
   compilerArgs = key;
   const previous = compiler;
   const created = (async () => {
+    const entries = libraries
+      ? await self.nimWasm.resolveLibraries(libraries, { cache: libraryCache })
+      : {};
+    const packages = Object.keys(entries).length ? entries : requestedPackages;
     // Compile through the Clang runtime this page already uses, rather than the second copy the
     // Nim IIFE carries inside it, so only one runtime is ever loaded.
     const built = await getToolchain();
@@ -74,9 +88,11 @@ const getCompiler = (compileArgs = []) => {
       baseUrl: ${JSON.stringify(nimWasmBaseUrl + 'assets/nim/')},
       toolchain: built,
       compileArgs,
+      packages,
     });
-    // compileArgs is fixed when a compiler is created, so a change builds a new one and
-    // releases the old; the shared toolchain belongs to this page and is not released.
+    // compileArgs, packages and libraries are fixed when a compiler is created, so a change
+    // builds a new one and releases the old; the shared toolchain belongs to this page and is
+    // not released.
     if (previous) previous.then((old) => old.dispose()).catch(() => undefined);
     return next;
   })();
@@ -93,11 +109,7 @@ const getCompiler = (compileArgs = []) => {
 addEventListener('message', async (event) => {
   const { id, code, input, options } = event.data;
   try {
-    const result = await (await getCompiler(options && options.compileArgs)).run(
-      code,
-      input,
-      options,
-    );
+    const result = await (await getCompiler(options)).run(code, input, options);
     postMessage({
       id,
       result: {

src/livecodes/languages/nim/lang-nim.ts

@@ -4,36 +4,61 @@ import { getLanguageCustomSettings } from '../utils';
 
 // Nim's JavaScript backend, for pages that want a document: the compiler emits one self-sufficient
 // file, so it is compiled with `execute: false` and the result is run in the result page, the same
-// shape as the TypeScript compiler's factory.
+// shape as the TypeScript compiler's factory. External packages are fetched on first import.
 export const nim: LanguageSpecs = {
   name: 'nim',
   title: 'Nim',
   compiler: {
     url: nimWasmBaseUrl + 'dist/nim-wasm.global.js',
     factory: () => {
-      const compilers: Record<'js' | 'c' | 'cpp', Promise<any> | null> = {
-        js: null,
-        c: null,
-        cpp: null,
-      };
-      const getCompiler = (target: 'js' | 'c' | 'cpp' = 'js') => {
-        if (!compilers[target]) {
-          const created: Promise<any> = (self as any).nimWasm.createCompiler({
-            target,
-            emit: 'single',
-            baseUrl: nimWasmBaseUrl + 'assets/nim/',
-          });
+      const compilers = new Map<string, Promise<any>>();
+      const libraryCache = new Map<string, Promise<any>>();
+      const getCompiler = async (
+        target: 'js' | 'c' | 'cpp' = 'js',
+        packages = true,
+        libraries?: unknown,
+      ) => {
+        const key = `${target}:${packages}:${JSON.stringify(libraries ?? null)}`;
+        let compiler = compilers.get(key);
+        if (!compiler) {
+          const created: Promise<any> = (async () => {
+            const entries = libraries
+              ? await (self as any).nimWasm.resolveLibraries(libraries, { cache: libraryCache })
+              : {};
+            return (self as any).nimWasm.createCompiler({
+              target,
+              emit: 'single',
+              baseUrl: nimWasmBaseUrl + 'assets/nim/',
+              // An object is merged with the bundled registry; a user library therefore enables the
+              // package loader even if `packages` was false.
+              packages: Object.keys(entries).length ? entries : packages,
+            });
+          })();
           // do not keep a failed boot (e.g. a transient network error)
           created.catch(() => {
-            compilers[target] = null;
+            if (compilers.get(key) === created) compilers.delete(key);
           });
-          compilers[target] = created;
+          compilers.set(key, created);
+          compiler = created;
         }
-        return compilers[target];
+        return compiler;
       };
       return async (code: string, { config }) => {
-        const target = getLanguageCustomSettings('nim', config)?.target || 'js';
-        const result = await (await getCompiler(target)).run(code, '', { execute: false });
+        const settings = getLanguageCustomSettings('nim', config) ?? {};
+        const target = settings.target || 'js';
+        const packages = settings.packages !== false;
+        let compiler: any;
+        try {
+          compiler = await getCompiler(target, packages, settings.libraries);
+        } catch (error) {
+          return {
+            code: '',
+            info: {
+              errors: [`Could not load Nim libraries: ${(error as Error)?.message ?? error}`],
+            },
+          };
+        }
+        const result = await compiler.run(code, '', { execute: false });
         if (!result.compiledCode) {
           return { code: '', info: { errors: result.errors || ['Nim produced no output.'] } };
         }

src/livecodes/vendors.ts

@@ -385,7 +385,7 @@ export const monacoVolarUrl = /* @__PURE__ */ getUrl(
 
 export const mustacheUrl = /* @__PURE__ */ getUrl('mustache@4.2.0/mustache.js');
 
-export const nimWasmBaseUrl = /* @__PURE__ */ getUrl('@live-codes/nim-wasm@0.5.0/');
+export const nimWasmBaseUrl = /* @__PURE__ */ getUrl('@live-codes/nim-wasm@0.6.0/');
 
 export const ninjaKeysUrl = /* @__PURE__ */ getUrl('@hatemhosny/ninja-keys@1.14.0/bundle/index.js');