feat(wrapper): generate the remote-plugin manifest at build time

README.md

@@ -714,11 +714,32 @@ can report a host pinned by Nix as out of date, or report errors when you are of
 the provider is broken, and the `npm install -g` style advice it gives does not apply: the host's
 version follows your nixpkgs.
 
-Two things these options do not do:
+#### Remote plugins
+
+With `true`, the remote plugins of that host in your lock are registered while nvimx builds, as
+with `programs.neovim`. There is no `:UpdateRemotePlugins` to run after a switch or a plugin
+update, and a lazy-loaded plugin's functions and commands work without the plugin being loaded.
+
+- Every plugin in the lock is registered, whatever its `lazy` / `cond` / `enabled` settings (again
+  as with `programs.neovim`). A `cond = false` plugin is still locked and in the plugin farm, so its
+  remote functions and commands exist even though lazy never loads the plugin's own Lua/Vimscript.
+  Remove the plugin from the spec and re-lock to drop them.
+- Your own manifest, `stdpath("data")/rplugin.vim` (what `:UpdateRemotePlugins` writes), is still
+  read. For the hosts nvimx registers, entries for paths under `/nix/store` are ignored in favour
+  of the build's, so an old manifest cannot shadow an updated plugin. Everything else in it is
+  kept.
+- `devPlugins` and dev/dir plugins are not registered by nvimx: run `:UpdateRemotePlugins` for
+  them, as before. (A working tree that itself lives under `/nix/store` would be treated as a
+  store plugin and ignored.)
+- A plugin whose registration fails at build time (for example, a package missing from
+  `extraPython3Packages`) is left out, with the error in the build log, instead of failing the
+  build. A package a remote plugin imports at registration time is needed at build time too.
+- Nothing is loaded with `--clean`, or when rplugin is disabled (`vim.g.loaded_remote_plugins` set
+  before plugins load, or `performance.rtp.disabled_plugins = { "rplugin" }`), matching Neovim.
+- `extraWrapperArgs` do not apply while the manifest is generated.
+
+One thing these options do not do:
 
-- **No remote-plugin manifest is generated.** Run `:UpdateRemotePlugins` yourself, and again after
-  every plugin update: the manifest records each plugin's path, which is a Nix store path that
-  changes with the lock.
 - **There is no `withPerl`.** Neovim's own discovery applies to Perl as it always has; put
   `vim.g.loaded_perl_provider = 0` in your `init.lua` to switch it off.
 

lua/nvimx/rplugin-gen.lua

@@ -0,0 +1,47 @@
+-- Runs under `nvim -l` inside the manifest derivation (nix/lib/rplugin-manifest.nix), and only
+-- when something was staged. arg[1] = the staging dir (one subdir per plugin, holding the rplugin/
+-- dirs of the hosts that are on), arg[2] = the raw manifest path (= $NVIM_RPLUGIN_MANIFEST).
+-- Writes the manifest to stdout, which the derivation appends after its header. Under `-l` only
+-- io.stdout reaches stdout (echomsg output such as "remote/host: ... registered plugins" goes to
+-- stderr), so host tracebacks and the warnings below stay in the build log.
+local stage, raw = arg[1], arg[2]
+
+for _, name in ipairs(vim.fn.readdir(stage)) do
+  vim.opt.rtp:append(stage .. "/" .. name)
+end
+vim.cmd("runtime! plugin/rplugin.vim")
+vim.cmd("UpdateRemotePlugins")
+
+-- A plugin whose host cannot load it (a missing Python package, say) is skipped by neovim and the
+-- run still succeeds. Say so instead of leaving it out silently. The patterns are the ones
+-- runtime/autoload/remote/host.vim passes to remote#host#Register for each host.
+local patterns = { python3 = "*", ruby = "*.rb", node = "*" }
+local f = assert(io.open(raw, "r"))
+local manifest = f:read("*a")
+f:close()
+
+-- the host's last message has no trailing newline
+io.stderr:write("\n")
+local missing = {}
+for host, pattern in pairs(patterns) do
+  for _, p in ipairs(vim.api.nvim_get_runtime_file("rplugin/" .. host .. "/" .. pattern, true)) do
+    local path = vim.fn.resolve(p)
+    if not manifest:find(path, 1, true) then
+      local plugin = vim.fn.fnamemodify(path, ":h:h:h:t")
+      io.stderr:write(
+        ("nvimx: %s (%s) did not register with the %s host and is left out of the manifest; see the host error above (a package missing from extraPython3Packages?)\n"):format(
+          path,
+          plugin,
+          host
+        )
+      )
+      table.insert(missing, '" not registered: ' .. path)
+    end
+  end
+end
+
+table.sort(missing)
+if #missing > 0 then
+  io.stdout:write(table.concat(missing, "\n"), "\n")
+end
+io.stdout:write(manifest)

lua/nvimx/rplugin-load.lua

@@ -0,0 +1,78 @@
+-- Loads nvimx's build-time remote-plugin manifest right after runtime/plugin/rplugin.vim (#85).
+-- dofile'd from the provider rc (nvimx-providers.lua) with
+--   { manifest = "<store path>", hosts = { "python3", ... }, store = "/nix/store/" }
+--
+-- The user's own manifest (stdpath("data")/rplugin.vim, what :UpdateRemotePlugins writes) is still
+-- read by runtime. Ours goes after it, and first every store-path registration of a host we
+-- generated is dropped, because a stale one wins even when ours is sourced later:
+--   - upstream remote#host#RegisterPlugin throws on a path registered twice (nixpkgs' patch turns
+--     the throw into a return), which would abort the rest of the user's file if ours went first;
+--   - pynvim imports plugin modules by basename, in registration order, so an old plugin version
+--     that is still in the store (or the same one) wins the module name and answers with old code;
+--   - a stale autocmd spec points at a handler that the new host never registered
+--     ("no request handler registered"), so the new handler never runs.
+-- Functions and commands would be redefined by the later registration, which is why only the
+-- three above need the removal. Registrations from outside the store (dev plugins) stay.
+return function(opts)
+  -- runtime's rplugin.vim `finish`es early when g:loaded_remote_plugins exists, but SourcePost
+  -- still fires after that; without this, a user who disabled rplugin that way would get it
+  -- switched back on. SourcePre sees the variable before the file runs.
+  local disabled_before = false
+  local pre = vim.api.nvim_create_autocmd("SourcePre", {
+    pattern = "*/plugin/rplugin.vim",
+    callback = function()
+      disabled_before = vim.g.loaded_remote_plugins ~= nil
+    end,
+  })
+  vim.api.nvim_create_autocmd("SourcePost", {
+    pattern = "*/plugin/rplugin.vim",
+    -- Not `once`: a plugin shipping its own plugin/rplugin.vim and sourced earlier must not use
+    -- the hook up. Returning true removes it after the real one has run.
+    callback = function()
+      if disabled_before then
+        pcall(vim.api.nvim_del_autocmd, pre)
+        return true
+      end
+      -- not runtime's rplugin.vim (that one sets this variable)
+      if vim.g.loaded_remote_plugins == nil then
+        return
+      end
+      -- runtime skips its manifest with --clean; so do we
+      if vim.tbl_contains(vim.v.argv, "--clean") then
+        pcall(vim.api.nvim_del_autocmd, pre)
+        return true
+      end
+      for _, host in ipairs(opts.hosts) do
+        local needle = 'Bootstrap("' .. host .. '", "' .. opts.store
+        -- PluginsForHost returns the live list, so filter() edits it in place. A lambda with
+        -- vim.fn.string() arguments, not a string expression: nested quotes broke that one.
+        vim.cmd(
+          ("call filter(remote#host#PluginsForHost(%s), {_, p -> stridx(p.path, %s) != 0})"):format(
+            vim.fn.string(host),
+            vim.fn.string(opts.store)
+          )
+        )
+        for _, au in ipairs(vim.api.nvim_get_autocmds({})) do
+          -- Vimscript-defined autocmds have no id, so nvim_del_autocmd cannot remove them: clear
+          -- their RPC_DEFINE_AUTOCMD_GROUP_* group instead.
+          if au.command and au.command:find(needle, 1, true) and au.group then
+            vim.api.nvim_clear_autocmds({ group = au.group })
+            pcall(vim.api.nvim_del_augroup_by_id, au.group)
+          end
+        end
+        for name, cmd in pairs(vim.api.nvim_get_commands({ builtin = false })) do
+          if cmd.definition:find(needle, 1, true) then
+            vim.api.nvim_del_user_command(name)
+          end
+        end
+      end
+      -- pcall: sourcing throws if init.lua started a host before lazy.setup ("already running")
+      local ok, err = pcall(vim.cmd.source, vim.fn.fnameescape(opts.manifest))
+      if not ok then
+        vim.notify("nvimx: " .. err, vim.log.levels.WARN)
+      end
+      pcall(vim.api.nvim_del_autocmd, pre)
+      return true -- delete this autocmd
+    end,
+  })
+end

nix/home-manager/default.nix

@@ -236,9 +236,14 @@ in
         you set there still wins. With false that does not bring the provider back by itself:
         init.lua has to unset vim.g.loaded_python3_provider as well.
 
-        nvimx does not generate a remote-plugin manifest. Run :UpdateRemotePlugins yourself, and
-        again after a plugin update: the manifest records the plugin's path, which is a Nix store
-        path that changes with the lock.
+        With true, nvimx also registers the Python remote plugins of the locked plugins at build
+        time, so there is nothing to run after a switch or a plugin update. Every plugin in the
+        lock is registered, whatever its lazy / cond / enabled settings, as with programs.neovim:
+        a cond = false plugin is still locked and in the farm, so its functions and commands
+        exist even though lazy never loads the plugin's Lua/Vimscript side. Plugins in
+        devPlugins (and the spec's dev/dir plugins) are left to :UpdateRemotePlugins. A plugin
+        whose registration fails (for example a package missing from extraPython3Packages) is
+        left out, with the error in the build log. See the README.
       '';
     };
 
@@ -259,6 +264,8 @@ in
 
         The list is still evaluated when withPython3 is not true -- that is how the warning is
         produced -- so a package that fails to evaluate is an error either way.
+
+        A package a remote plugin imports at registration time is needed at build time too.
       '';
     };
 
@@ -279,6 +286,9 @@ in
         a ruby you already have stays in front), which is what :checkhealth looks for. Unlike
         nixpkgs' wrapper, nvimx does not set GEM_HOME: the host runs without it, and it would be
         exported to every process neovim starts. Add it with extraWrapperArgs if you need it.
+
+        With true, Ruby remote plugins in the lock are registered at build time, as for
+        withPython3.
       '';
     };
 
@@ -299,6 +309,9 @@ in
         there. As with programs.neovim, appending it also makes `node` available to plugins
         that run it themselves. The host therefore runs under the first node on PATH -- yours,
         if you have one; put pkgs.nodejs in extraPackages to pin it.
+
+        With true, JavaScript remote plugins in the lock are registered at build time, as for
+        withPython3.
       '';
     };
 

nix/lib/make-env.nix

@@ -20,6 +20,8 @@
   # arguments rather than a `providers = { ... }` attrset, for the reason devPlugins / devPath
   # give below: they map 1:1 onto the module's options, which are flat because
   # programs.neovim's are. Threaded straight to the wrapper, which hands them to providers.nix.
+  # With true, providers.nix also registers the locked plugins' remote plugins for that host at
+  # build time (#85); the farm is added to that hand-over below.
   withPython3 ? null,
   withRuby ? null,
   withNodeJs ? null,
@@ -183,9 +185,13 @@ let
       withRuby
       withNodeJs
       extraPython3Packages
+      farm
       vimAlias
       viAlias
       ;
+    # Dev plugins (devPlugins and localPlugins) are left to :UpdateRemotePlugins: their working-tree
+    # paths are not store paths, so the user's own registrations of them must keep winning (#85).
+    rpluginExclude = builtins.attrNames devDirs;
   };
 in
 {

nix/lib/providers.nix

@@ -1,6 +1,7 @@
 # Remote-plugin provider hosts (python3 / ruby / node) for the wrapped neovim (#73).
 # Returns what nix/lib/wrapper.nix needs and nothing else: `rc`, a Lua file to run with --cmd
-# (null when there is nothing to say), and `wrapperArgs`, a flat list of makeWrapper arguments.
+# (null when there is nothing to say), `wrapperArgs`, a flat list of makeWrapper arguments, and
+# `manifest`, the build-time remote-plugin manifest (#85; null unless a host is true, see below).
 #
 # Each with* argument has three states, and the third is why this file writes its own lines
 # instead of reusing wrapNeovimUnstable's providerLuaRc, which only knows two:
@@ -11,13 +12,20 @@
 #   false -> vim.g.loaded_<prog>_provider = 0: the provider is off, and neovim skips the search.
 # true and false mean what they mean in home-manager's programs.neovim, so a value carried over
 # from there keeps its meaning; null is the state programs.neovim does not have.
+#
+# true additionally registers the remote plugins of that host at build time (nix/lib/rplugin-manifest.nix)
+# and has the rc load the result. null and false change nothing, so those wrappers stay as they were.
 { pkgs }:
 {
   package,
   withPython3 ? null,
   withRuby ? null,
   withNodeJs ? null,
   extraPython3Packages ? (_: [ ]),
+  # For the remote-plugin manifest (#85): the plugin farm to register from, and the farm entries
+  # left to :UpdateRemotePlugins (dev plugins). Without a farm there is no manifest.
+  farm ? null,
+  rpluginExclude ? [ ],
 }:
 let
   inherit (pkgs) lib;
@@ -84,13 +92,43 @@ let
   # python3 needs nothing: g:python3_host_prog is executed directly.
   runtimeDeps =
     lib.optional (withRuby == true) rubyEnv ++ lib.optional (withNodeJs == true) pkgs.nodejs;
+
+  # The hosts that are on, i.e. the ones whose remote plugins get registered at build time (#85).
+  # null and false register nothing and leave the rc and the wrapper exactly as they were.
+  manifestHosts = lib.concatLists [
+    (lib.optional (withPython3 == true) "python3")
+    (lib.optional (withRuby == true) "ruby")
+    (lib.optional (withNodeJs == true) "node")
+  ];
+  # Today's rc: the vim.g.*_host_prog lines. Also what the manifest derivation runs with --cmd.
+  hostRc = pkgs.writeText "nvimx-providers.lua" (lib.concatStringsSep "\n" lines + "\n");
+  manifest =
+    if farm != null && manifestHosts != [ ] then
+      import ./rplugin-manifest.nix { inherit pkgs; } {
+        inherit package farm hostRc;
+        hosts = manifestHosts;
+        path = runtimeDeps;
+        exclude = rpluginExclude;
+      }
+    else
+      null;
+  loadLine = "dofile(${lib.generators.toLua { } "${../../lua/nvimx/rplugin-load.lua}"})(${
+    lib.generators.toLua { } {
+      manifest = "${manifest}";
+      hosts = manifestHosts;
+      store = builtins.storeDir + "/";
+    }
+  })";
 in
 {
+  inherit manifest;
   rc =
     if lines == [ ] then
       null
+    else if manifest == null then
+      hostRc
     else
-      pkgs.writeText "nvimx-providers.lua" (lib.concatStringsSep "\n" lines + "\n");
+      pkgs.writeText "nvimx-providers.lua" (lib.concatStringsSep "\n" (lines ++ [ loadLine ]) + "\n");
   wrapperArgs = lib.optionals (runtimeDeps != [ ]) [
     "--suffix"
     "PATH"

nix/lib/rplugin-manifest.nix

@@ -0,0 +1,63 @@
+# Build-time remote-plugin manifest (#85): what :UpdateRemotePlugins would write for the plugins
+# in the farm, produced while nvimx builds so that nothing has to run after a switch or an update.
+# See lua/nvimx/rplugin-gen.lua for what runs inside neovim, and lua/nvimx/rplugin-load.lua for how
+# the result is loaded at run time.
+#   package  the neovim that generates (not the final wrapper: the wrapper's rc depends on this
+#            derivation, so using it here would be circular)
+#   farm     the plugin farm; every entry is a candidate
+#   hosts    host names to register for ("python3" / "ruby" / "node"); only these are staged, so
+#            a host the user did not turn on is never searched for or started
+#   hostRc   today's provider rc (the vim.g.*_host_prog lines), run with --cmd
+#   path     what the hosts need on PATH while generating (node must be there)
+#   exclude  farm entries to leave alone (dev plugins stay with :UpdateRemotePlugins)
+{ pkgs }:
+{
+  package,
+  farm,
+  hosts,
+  hostRc,
+  path ? [ ],
+  exclude ? [ ],
+}:
+let
+  inherit (pkgs) lib;
+in
+pkgs.runCommand "nvimx-rplugin-manifest.vim" { } ''
+  export HOME=$TMPDIR XDG_CONFIG_HOME=$TMPDIR/config XDG_DATA_HOME=$TMPDIR/data \
+         XDG_STATE_HOME=$TMPDIR/state XDG_CACHE_HOME=$TMPDIR/cache
+  ${lib.optionalString (path != [ ]) "export PATH=${lib.makeBinPath path}:$PATH"}
+
+  # Which plugins have something to register is decided here, at build time: reading the farm at
+  # evaluation time would be IFD. Each plugin is staged as a directory holding only the rplugin/
+  # dirs of hosts that are on, symlinked from the farm. neovim resolves symlinks when it records a
+  # path, so the manifest names the plugin's own store path, the same string a user's own
+  # :UpdateRemotePlugins records.
+  mkdir stage
+  for dir in ${farm}/*; do
+    name=$(basename "$dir")
+    skip=
+    for ex in lazy.nvim ${lib.escapeShellArgs exclude}; do
+      if [ "$name" = "$ex" ]; then skip=1; fi
+    done
+    if [ -n "$skip" ]; then continue; fi
+    for host in ${lib.escapeShellArgs hosts}; do
+      if [ -d "$dir/rplugin/$host" ]; then
+        mkdir -p "stage/$name/rplugin"
+        ln -s "$dir/rplugin/$host" "stage/$name/rplugin/$host"
+      fi
+    done
+  done
+
+  echo '" Generated by nvimx at build time (#85) for: ${lib.concatStringsSep " " hosts}' > $out
+  # Nothing staged: no neovim run at all, and the manifest is the header only.
+  if [ -n "$(ls -A stage)" ]; then
+    # Upstream's variable, read when runtime/plugin/rplugin.vim is sourced; the raw manifest
+    # lands here and rplugin-gen.lua copies it to stdout after checking it.
+    export NVIM_RPLUGIN_MANIFEST=$TMPDIR/raw.vim
+    # -u NORC, not -u NONE: with NONE no plugin is loaded and :UpdateRemotePlugins is not defined.
+    # timeout: a host that hangs would otherwise hang the build; a failing plugin is not fatal
+    # (the plugin is left out and the error stays in the log), but a hang or a crash is.
+    timeout 600 ${package}/bin/nvim -u NORC -i NONE -n --cmd "luafile ${hostRc}" \
+      -l ${../../lua/nvimx/rplugin-gen.lua} "$PWD/stage" "$TMPDIR/raw.vim" >> $out
+  fi
+''