Implement recursive path building in getSrc

${l.S ...} now tries to build its arguments when they do not exist, yet. It
behaves like a redo-ifchange, now.

builder.nix

@@ -1,4 +1,4 @@
-buildArgs@{ lib ? import ./lib.nix {}, nixpkgs ? <nixpkgs>, pkgs ? import nixpkgs {}, root, cwd, filter ? [] }:
+buildArgs@{ nixpkgs ? <nixpkgs>, pkgs ? import nixpkgs {}, lib ? import ./lib.nix { inherit pkgs; }, root, filter ? [] }:
 let
   inherit (lib) searchPath whichdo d1 d2 getSrc;
 
@@ -19,12 +19,11 @@ let
       # scripts.
       funcPkgs = {
         __functor = self: arg: resolve arg;
-        src = getSrc root rcwd filter id;
+        src = getSrc root rcwd filter resolve id;
-        inherit pkgs cwd rcwd d1 d2;
+        inherit pkgs rcwd d1 d2;
         # Convenience function that calculates the source tree, links it to src
         # and cd's to the location of the current builder.
-        S = getSrc root rcwd filter (src:
+        S = getSrc root rcwd filter resolve (src: ''
-          ''
           ln -s ${src} src
           cd src/${rcwd}
         '');
@@ -33,6 +32,7 @@ let
       _builder = let
         imported = (tryEval (let
           # Needs to be done as nix chokes on empty files rn.
+          # Potential source of memory exhaustion for untrusted very large *.do files!
           content = readFile builder;
         in assert content != "" && (! (hasPrefix "#!" content)); import builder));
       in if imported.success

lib.nix

@@ -1,10 +1,14 @@
-{ lib ? import <nixpkgs/lib> }:
+{ pkgs ? import <nixpkgs> {} }:
 let
-  inherit (builtins) baseNameOf dirOf length genList pathExists filterSource
+  lib = pkgs.lib;
-  isString match replaceStrings stringLength;
+  inherit (builtins) baseNameOf dirOf length genList pathExists isString match
-  inherit (lib) take splitString concatStringsSep last foldl foldr head tail
+  replaceStrings;
-  singleton removePrefix hasSuffix removeSuffix flatten crossLists reverseList
+  inherit (lib) take splitString last foldl foldr head tail removePrefix
-  all any unique hasPrefix;
+  hasSuffix removeSuffix flatten crossLists reverseList all any unique hasPrefix
+  cleanSourceWith;
+
+  inherit (pkgs) runCommand symlinkJoin;
+
   butlast = list: take (length list - 1) list;
 
   doFileSuffix = "do";
@@ -83,14 +87,27 @@ let
   # The source tree resolver combinator.
   # Takes the source root and cwd of the current builder and a list of paths
   # that shall always be rejected.
+  # Also takes a resolver to derive unknown paths and a function to which the
+  # resulting set is applied to.
   #
   # Returns a filter that accepts zero to n paths relative to cwd. Captures the
   # whole source tree when called with zero arguments.
-  getSrc = root: rcwd: rejected: outFunc:
+  getSrc = root: rcwd: rejected: resolver: outFunc:
     captureFunc (resolveSrc root rcwd) (srcs: let
 
       rejector = path: (all (x: path != x) rejected);
 
+      # Filter non-existant sources and build them with resolver.
+      # Recreate the source directory structure.
+      nonExistantSrcs = builtins.filter (x: !builtins.pathExists x) srcs;
+      builtSrcs = map (s: let
+        relPath = removePrefix (root + "/") s;
+        filePath = resolver s;
+      in runCommand relPath { preferLocalBuild = true; allowSubstitutes = false; } ''
+         mkdir -p "$out/${dirOf relPath}"
+         ln -s "${filePath}" "$out/${relPath}"
+      '') nonExistantSrcs;
+
       # If a src in srcs is a prefix to a that (it must be a dir, then, or the
       # file itself), allow all sub-paths.
       srcFilter = path:
@@ -103,13 +120,21 @@ let
       srcDirFilter = path: if srcs == []
                         then true # No explicit srcs means: Take the whole source tree!
                         else (any (x: path == x) (_srcs));
+
+      # Filter the sources by our defined criteria and create a source tree
+      # derivation.
       filter = (path: type:
         (baseNameOf path != ".git") &&
         (baseNameOf path != ".envrc") &&
         (rejector path) &&
         ((srcFilter path) ||
          (srcDirFilter path)));
-    in outFunc (builtins.filterSource filter root));
+      filteredSrcs = cleanSourceWith { inherit filter; src = root; };
+
+      # Merge the filtered sources and the newly built dependencies.
+      mergedSrcs = symlinkJoin { name = "srcs"; paths = [ filteredSrcs ] ++ builtSrcs; };
+
+    in outFunc mergedSrcs);
 
   self = {
     inherit doFileSuffix whichdo d1 d2 getSrc;

nixredo

@@ -27,7 +27,7 @@ out=$(nix -vL --show-trace build ${NIXREDO_ROOT:+--store "$NIXREDO_ROOT"} \
           --option auto-optimise-store true --option substituters daemon \
           --option keep-failed true --option keep-outputs true \
           --impure --json --no-link \
-          --expr " import @buildernix@ { lib = import @libnix@ {}; root =\"$S\"; cwd = \"$PWD\"; filter = [ \"$filter\" \"$f\" ]; } \"$f\"" |
+          --expr " import @buildernix@ { lib = import @libnix@ {}; root =\"$S\"; filter = [ \"$filter\" \"$f\" ]; } \"$f\"" |
           @jq@ -r '.[0].outputs.out')
 [ -n "$out" ] || exit 127
 ln -s "${NIXREDO_ROOT+$NIXREDO_ROOT}$out" "$3"