From 3f4a79613d027433736a673a9ccffe1985b104b4 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 24 Jan 2022 11:46:15 +0100 Subject: [PATCH 1/7] Add failing test case for directory with ignored contents In git, the entire directory does not exist In gitignoresource, the empty directory does exist --- tests/testdata.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/testdata.nix b/tests/testdata.nix index 29a5100..af1b08e 100644 --- a/tests/testdata.nix +++ b/tests/testdata.nix @@ -54,6 +54,9 @@ let mkdir 12-empty-dir mkdir 12-not-empty-dir touch 12-not-empty-dir/just-a-regular-file + + mkdir 13-dir-with-ignored-contents + touches 13-dir-with-ignored-contents/{foo,bar} ); } create-tree "$1" @@ -106,6 +109,8 @@ let # two bracketed classes in one rule 7-brackets/*- [Bb]ackup ([0-9]).rdl + + 13-dir-with-ignored-contents/* ''; ignoresAux = "/9-expected/*filepath\n"; From 49e5d6daf3a4531fc6736c8b74032e26eae8d145 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Sep 2023 11:20:25 +0200 Subject: [PATCH 2/7] Format --- flake.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index e258cb1..a6217ea 100644 --- a/flake.nix +++ b/flake.nix @@ -6,8 +6,9 @@ inherit (nixpkgs) lib; }; - overlay = final: prev: import ./default.nix { - inherit (prev) lib; - }; + overlay = final: prev: + import ./default.nix { + inherit (prev) lib; + }; }; } From 2eddf71196502466ef4f8e6c6380c08ca5e1c048 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Sep 2023 11:21:31 +0200 Subject: [PATCH 3/7] flake.nix: description: does not have to be git --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a6217ea..39ff8f4 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "Nix functions for filtering local git sources"; + description = "Nix functions for filtering local sources"; outputs = { self, nixpkgs }: { lib = import ./default.nix { From 1a2cd1693215e33aa8186b7b1754ed72342ff174 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Sep 2023 11:40:12 +0200 Subject: [PATCH 4/7] flake.nix: Wire the checks --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 39ff8f4..cadc52a 100644 --- a/flake.nix +++ b/flake.nix @@ -10,5 +10,7 @@ import ./default.nix { inherit (prev) lib; }; + + checks.x86_64-linux = import ./tests { pkgs = nixpkgs.legacyPackages.x86_64-linux; }; }; } From 17ab30e0a9fc11aba444ea59fb2ecca41329fe36 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Sep 2023 11:40:26 +0200 Subject: [PATCH 5/7] flake.lock: Update --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index de9f824..4552b4d 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1632846328, - "narHash": "sha256-sFi6YtlGK30TBB9o6CW7LG9mYHkgtKeWbSLAjjrNTX0=", + "lastModified": 1666603677, + "narHash": "sha256-apAEIj+z1iwMaMJ4tB21r/VTetfGDLDzuhXRHJknIAU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2b71ddd869ad592510553d09fe89c9709fa26b2b", + "rev": "074da18a72269cc5a6cf444dce42daea5649b2fe", "type": "github" }, "original": { From b5594318e5b8118c302b82099b75cdfb25f6ea92 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Sep 2023 16:58:51 +0200 Subject: [PATCH 6/7] Fix dir with ignored contents bug --- find-files.nix | 23 ++++++++++----- lib/path.nix | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ tests/x.bar | 0 tests/x.foo | 0 tests/x.qux | 0 5 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 lib/path.nix create mode 100644 tests/x.bar create mode 100644 tests/x.foo create mode 100644 tests/x.qux diff --git a/find-files.nix b/find-files.nix index 5d7c560..e52fc06 100644 --- a/find-files.nix +++ b/find-files.nix @@ -2,6 +2,7 @@ let parse-ini = import ./parse-git-config.nix { inherit lib; }; parse-gitignore = import ./rules.nix { inherit lib; }; + path = import ./lib/path.nix { inherit lib; }; in rec { inherit (builtins) dirOf baseNameOf abort split hasAttr readFile readDir pathExists; @@ -27,14 +28,20 @@ rec { lib.optional (extraRules != null) { contextDir = basePath; rules = extraRules; }; patternsBelowP = findPatternsTree extraRules2 basePath; basePathStr = toString basePath; - in - path: type: let - localDirPath = removePrefix basePathStr (toString (dirOf path)); - localDirPathElements = splitString "/" localDirPath; - patternResult = parse-gitignore.runFilterPattern (getPatterns patternsBelowP localDirPathElements)."/patterns" path type; - nonempty = any (nodeName: gitignoreFilter (basePath + "/${nodeName}") != false) - (attrNames (readDir path)); - in patternResult && (type == "directory" -> nonempty); + rawFilter = + path: type: let + localDirPath = removePrefix basePathStr (toString (dirOf path)); + localDirPathElements = splitString "/" localDirPath; + patternResult = parse-gitignore.runFilterPattern (getPatterns patternsBelowP localDirPathElements)."/patterns" path type; + contents = readDir path; + nonempty = localDirPathElements == [] || any (nodeName: memoFilter (path + "/${nodeName}") != false) + (attrNames contents); + in patternResult && (type == "directory" -> nonempty); + memoFilter = + path.memoize rawFilter + (p: throw "Could not find path ${toString p} in memo table. Did path or filtering semantics change?") + basePath; + in path: type: memoFilter path; getPatterns = patternTree: pathElems: diff --git a/lib/path.nix b/lib/path.nix new file mode 100644 index 0000000..dcd5646 --- /dev/null +++ b/lib/path.nix @@ -0,0 +1,79 @@ +{ lib }: +let + inherit (lib) filter head mapAttrs tail; + + # absolutePathComponentsBetween : PathOrString -> PathOrString -> [String] + # absolutePathComponentsBetween ancestor descendant + # + # Returns the path components that form the path from ancestor to descendant. + # Will not return ".." components, which is a feature. Throws when ancestor + # and descendant arguments aren't in said relation to each other. + # + # Example: + # + # absolutePathComponentsBetween /a /a/b/c == ["b" "c"] + # absolutePathComponentsBetween /a/b/c /a/b/c == [] + absolutePathComponentsBetween = + ancestor: descendant: + let + a' = /. + ancestor; + go = d: + if a' == d + then [] + else if d == /. + then throw "absolutePathComponentsBetween: path ${toString ancestor} is not an ancestor of ${toString descendant}" + else go (dirOf d) ++ [(baseNameOf d)]; + in + go (/. + descendant); + + /* + Memoize a function that takes a path argument. + Example: + analyzeTree = dir: + let g = memoizePathFunction (p: t: expensiveFunction p t) (p: {}) dir; + in presentExpensiveData g; + Type: + memoizePathFunction :: (Path -> Type -> a) -> (Path -> a) -> Path -> (Path -> a) + */ + memoize = + # Function to memoize + f: + # What to return when a path does not exist, as a function of the path + missing: + # Filesystem location below which the returned function is defined. `/.` may be acceptable, but a path closer to the data of interest is better. + root: + + let + makeTree = dir: type: { + value = f dir type; + inherit type; + children = + if type == "directory" + then mapAttrs + (key: type: makeTree (dir + "/${key}") type) + (builtins.readDir dir) + else {}; + }; + + # This is where the memoization happens + tree = makeTree root ( + # We can't query the type of a store path in Nix without readFileType in + # pure mode, so we assume. + "directory" + ); + + lookup = notFound: list: subtree: + if list == [] + then subtree.value + else if subtree.children ? ${head list} + then lookup notFound (tail list) subtree.children.${head list} + else notFound; + in + path: lookup + (missing path) + (absolutePathComponentsBetween root path) + tree; +in +{ + inherit memoize absolutePathComponentsBetween; +} diff --git a/tests/x.bar b/tests/x.bar new file mode 100644 index 0000000..e69de29 diff --git a/tests/x.foo b/tests/x.foo new file mode 100644 index 0000000..e69de29 diff --git a/tests/x.qux b/tests/x.qux new file mode 100644 index 0000000..e69de29 From 02c007b372999722e9435208d005fbeeca305f7a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Sep 2023 17:16:10 +0200 Subject: [PATCH 7/7] Reminder to port the tests --- lib/path.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/path.nix b/lib/path.nix index dcd5646..fb37f11 100644 --- a/lib/path.nix +++ b/lib/path.nix @@ -14,6 +14,7 @@ let # absolutePathComponentsBetween /a /a/b/c == ["b" "c"] # absolutePathComponentsBetween /a/b/c /a/b/c == [] absolutePathComponentsBetween = + # TODO: port the tests from https://github.com/NixOS/nixpkgs/pull/112083 ancestor: descendant: let a' = /. + ancestor; @@ -43,6 +44,8 @@ let # Filesystem location below which the returned function is defined. `/.` may be acceptable, but a path closer to the data of interest is better. root: + # TODO: port the tests from https://github.com/NixOS/nixpkgs/pull/112083 + let makeTree = dir: type: { value = f dir type;