feat(include-qualified): support dynamic directory mappings

doc/changes/added/13691.md

@@ -1,3 +1,4 @@
 - Add `(dirs ...)` to `(include_subdirs (mode qualified) ...)`, allowing
-  qualified source directories to be renamed in the module path (#13691, fixes
-  #7605, @anmonteiro).
+  qualified source directories to be renamed in the module path. Mappings can
+  use variables, including reads from generated files outside the qualified
+  group (#13691, #16420, fixes #7605, @anmonteiro).

src/dune_rules/dir_contents.ml

@@ -326,7 +326,13 @@ end = struct
     type t
 
     val empty : t
-    val create : dir:Path.Build.t -> File_binding.Unexpanded.t list -> t
+
+    val expand
+      :  Super_context.t
+      -> dir:Path.Build.t
+      -> File_binding.Unexpanded.t list
+      -> t Memo.t
+
     val translate : t -> Filename.t list -> Filename.t list
   end = struct
     type binding =
@@ -361,28 +367,16 @@ end = struct
          | segments -> segments)
     ;;
 
-    let literal sw =
-      let loc = String_with_vars.loc sw in
-      match String_with_vars.text_only sw with
-      | Some text -> loc, text
-      | None ->
-        User_error.raise
-          ~loc
-          [ Pp.text "Variables are not supported in directory mappings." ]
-    ;;
-
-    let of_binding ~dir binding =
-      let src_loc, src = literal (File_binding.Unexpanded.src binding) in
-      match File_binding.Unexpanded.dst binding with
+    let expand_binding ~dir binding =
+      match File_binding.Expanded.dst_with_loc binding with
       | None -> None
-      | Some dst ->
-        let dst_loc, dst = literal dst in
+      | Some (dst_loc, dst) ->
         let root = Path.Build.local dir in
         let src =
           descendant_segments
-            ~loc:src_loc
+            ~loc:(File_binding.Expanded.src_loc binding)
             ~what:"The source directory"
-            (Path.Build.relative dir src |> Path.Build.local)
+            (File_binding.Expanded.src binding |> Path.Build.local)
             ~of_:root
         in
         let dst =
@@ -403,8 +397,17 @@ end = struct
         else Some (Path.Local.of_comps src, (dst_loc, { src; dst }))
     ;;
 
-    let create ~dir dirs =
-      List.filter_map dirs ~f:(of_binding ~dir)
+    let expand sctx ~dir dirs =
+      let* expand =
+        let+ expander = Super_context.expander sctx ~dir in
+        Expander.expand_str expander
+      in
+      let+ bindings =
+        Memo.parallel_map dirs ~f:(fun binding ->
+          File_binding_expand.expand binding ~dir ~f:(fun sw ->
+            Action_builder.evaluate_and_collect_facts (expand sw) >>| fst))
+      in
+      List.filter_map bindings ~f:(expand_binding ~dir)
       |> Path.Local.Map.of_list_reducei
            ~f:(fun src ((_, first) as previous) (loc, second) ->
              if List.equal Filename.equal first.dst second.dst
@@ -455,13 +458,12 @@ end = struct
       let loc, qualif_mode = qualification in
       loc, Include_subdirs.Include qualif_mode
     in
-    let dir_renames =
+    let+ dir_renames =
       match snd qualification with
-      | Unqualified | Qualified { dirs = [] } -> Dir_renames.empty
-      | Qualified { dirs } -> Dir_renames.create ~dir dirs
+      | Unqualified | Qualified { dirs = [] } -> Memo.return Dir_renames.empty
+      | Qualified { dirs } -> Dir_renames.expand sctx ~dir dirs
     in
     let loc = loc_of_dune_file source_dir in
-    let+ components = components in
     let contents =
       Memo.lazy_
         ~name:"group-dir-contents"
@@ -481,6 +483,7 @@ end = struct
                           ~src_dir:(Dune_file.dir dune_file)
                           ~dir)
                  (fun () ->
+                    let* components = components in
                     Memo.parallel_map
                       components
                       ~f:(fun { dir; path_to_group_root; source_dir; stanzas } ->

test/blackbox-tests/test-cases/include-qualified/nested-renamed-group-interface.t

@@ -302,9 +302,8 @@ Repeating a generator for the same source still reports conflicting rules.
   Leaving directory 'lexer'
   [1]
 
-Dynamic mappings should build a generated mapping file outside the qualified
+Dynamic mappings build a generated mapping file outside the qualified
 group and update the namespace when its input changes, without cleaning.
-This currently fails because directory mappings require literal paths.
 
   $ mkdir -p dynamic/config dynamic/lib/internal
   $ cat >dynamic/dune-project <<EOF
@@ -325,36 +324,20 @@ This currently fails because directory mappings require literal paths.
 
   $ printf public >dynamic/config/mapping.in
   $ dune build --root=dynamic '%{cmi:lib/Public.Leaf}'
-  Entering directory 'dynamic'
-  File "lib/dune", line 3, characters 20-45:
-  3 |  (dirs (internal as %{read:../config/mapping})))
-                          ^^^^^^^^^^^^^^^^^^^^^^^^^
-  Error: Variables are not supported in directory mappings.
-  Leaving directory 'dynamic'
-  [1]
 
   $ printf exposed >dynamic/config/mapping.in
   $ dune build --root=dynamic '%{cmi:lib/Exposed.Leaf}'
-  Entering directory 'dynamic'
-  File "lib/dune", line 3, characters 20-45:
-  3 |  (dirs (internal as %{read:../config/mapping})))
-                          ^^^^^^^^^^^^^^^^^^^^^^^^^
-  Error: Variables are not supported in directory mappings.
-  Leaving directory 'dynamic'
-  [1]
 
-After remapping, the old namespace should no longer exist.
+After remapping, the old namespace no longer exists.
 
   $ dune build --root=dynamic '%{cmi:lib/Public.Leaf}'
   Entering directory 'dynamic'
-  File "lib/dune", line 3, characters 20-45:
-  3 |  (dirs (internal as %{read:../config/mapping})))
-                          ^^^^^^^^^^^^^^^^^^^^^^^^^
-  Error: Variables are not supported in directory mappings.
+  File "command line", line 1, characters 0-22:
+  Error: Module Public.Leaf does not exist.
   Leaving directory 'dynamic'
   [1]
 
-Environment variables on either side are also rejected for now.
+Environment variables can occur on either side of a mapping.
 
   $ export TEST_RENAME=public TEST_SOURCE=internal
   $ cat >dynamic/lib/dune <<EOF
@@ -364,13 +347,6 @@ Environment variables on either side are also rejected for now.
   > (library (name renamed))
   > EOF
   $ dune build --root=dynamic '%{cmi:lib/Public.Leaf}'
-  Entering directory 'dynamic'
-  File "lib/dune", line 3, characters 20-45:
-  3 |  (dirs (internal as %{env:TEST_RENAME=unused})))
-                          ^^^^^^^^^^^^^^^^^^^^^^^^^
-  Error: Variables are not supported in directory mappings.
-  Leaving directory 'dynamic'
-  [1]
 
   $ cat >dynamic/lib/dune <<EOF
   > (include_subdirs
@@ -379,16 +355,8 @@ Environment variables on either side are also rejected for now.
   > (library (name renamed))
   > EOF
   $ dune build --root=dynamic '%{cmi:lib/Public.Leaf}'
-  Entering directory 'dynamic'
-  File "lib/dune", line 3, characters 8-33:
-  3 |  (dirs (%{env:TEST_SOURCE=unused} as public)))
-              ^^^^^^^^^^^^^^^^^^^^^^^^^
-  Error: Variables are not supported in directory mappings.
-  Leaving directory 'dynamic'
-  [1]
 
-A dynamic destination can conflict with a literal mapping. For now,
-expansion is rejected before checking for that conflict.
+Expanded destinations must not conflict with literal mappings.
 
   $ cat >dynamic/lib/dune <<EOF
   > (include_subdirs
@@ -402,7 +370,7 @@ expansion is rejected before checking for that conflict.
   File "lib/dune", line 4, characters 20-45:
   4 |        (internal as %{read:../config/mapping})))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^
-  Error: Variables are not supported in directory mappings.
+  Error: The directory internal is mapped to both public and exposed.
   Leaving directory 'dynamic'
   [1]