Merge pull request #289 from hercules-ci/formatter-and-touchups

Fix `formatter` laziness and add touchups module

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -4,3 +4,6 @@ result-*
 
 # pre-commit-hooks.nix
 .pre-commit-config.yaml
+
+# direnv
+.direnv/

dev/flake-module.nix

@@ -4,6 +4,7 @@
   imports = [
     inputs.pre-commit-hooks-nix.flakeModule
     inputs.hercules-ci-effects.flakeModule # herculesCI attr
+    inputs.nix-unit.modules.flake.default
   ];
   systems = [ "x86_64-linux" "aarch64-darwin" ];
 
@@ -21,6 +22,7 @@
 
     devShells.default = pkgs.mkShell {
       nativeBuildInputs = [
+        config.nix-unit.package
         pkgs.nixpkgs-fmt
         pkgs.hci
       ];
@@ -36,9 +38,15 @@
       };
     };
 
-    checks.eval-tests =
+    checks.perSystem-memoize = pkgs.callPackage ./tests/perSystem-memoize.nix {
-      let tests = import ./tests/eval-tests.nix { flake-parts = self; };
+      flake-parts = self;
-      in tests.runTests pkgs.emptyFile // { internals = tests; };
+    };
+
+    nix-unit.tests = import ./tests/nix-unit.nix { flake-parts = self; };
+
+    # nix-unit evaluates the flake, which triggers the dev partition via
+    # flake-compat, requiring network to fetch dev inputs.
+    nix-unit.allowNetwork = true;
 
   };
   flake = {

dev/flake.lock

@@ -76,6 +76,49 @@
         "type": "github"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-unit",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1737420293,
+        "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-unit": {
+      "inputs": {
+        "nix-github-actions": "nix-github-actions",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1777114263,
+        "narHash": "sha256-vZfRXBDC9FTO2Vpz8TroVMqOYqp+hcVk6Nwx6+kRN1Q=",
+        "owner": "nix-community",
+        "repo": "nix-unit",
+        "rev": "02e6597889281382303cbb7973748f04f2940e1a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-unit",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1775710090,
@@ -132,9 +175,31 @@
     "root": {
       "inputs": {
         "hercules-ci-effects": "hercules-ci-effects",
+        "nix-unit": "nix-unit",
         "nixpkgs": "nixpkgs_2",
         "pre-commit-hooks-nix": "pre-commit-hooks-nix"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-unit",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1775636079,
+        "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

dev/flake.nix

@@ -10,6 +10,8 @@
     pre-commit-hooks-nix.url = "github:cachix/pre-commit-hooks.nix";
     pre-commit-hooks-nix.inputs.nixpkgs.follows = "nixpkgs";
     hercules-ci-effects.url = "github:hercules-ci/hercules-ci-effects";
+    nix-unit.url = "github:nix-community/nix-unit";
+    nix-unit.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs = { ... }:

dev/tests/eval-tests.nix

@@ -1,361 +0,0 @@
-# Run with
-#
-#     nix build .#checks.x86_64-linux.eval-tests
-
-{ flake-parts }:
-rec {
-  nixpkgs = flake-parts.inputs.nixpkgs;
-  f-p-lib = flake-parts.lib;
-  inherit (f-p-lib) mkFlake;
-  inherit (flake-parts.inputs.nixpkgs-lib) lib;
-
-  pkg = system: name:
-    derivation
-      {
-        name = name;
-        builder = "no-builder";
-        system = system;
-      }
-    // {
-      meta = {
-        mainProgram = name;
-      };
-    };
-
-  empty = mkFlake
-    { inputs.self = { }; }
-    {
-      systems = [ ];
-    };
-
-  emptyExposeArgs = mkFlake
-    { inputs.self = { outPath = "the self outpath"; }; }
-    ({ config, moduleLocation, ... }: {
-      flake = {
-        inherit moduleLocation;
-      };
-    });
-
-  emptyExposeArgsNoSelf = mkFlake
-    { inputs.self = throw "self won't be available in case of some errors"; }
-    ({ config, moduleLocation, ... }: {
-      flake = {
-        inherit moduleLocation;
-      };
-    });
-
-  example1 = mkFlake
-    { inputs.self = { }; }
-    {
-      systems = [ "a" "b" ];
-      perSystem = { config, system, ... }: {
-        packages.hello = pkg system "hello";
-        apps.hello.program = config.packages.hello;
-      };
-    };
-
-  packagesNonStrictInDevShells = mkFlake
-    { inputs.self = packagesNonStrictInDevShells; /* approximation */ }
-    {
-      systems = [ "a" "b" ];
-      perSystem = { system, self', ... }: {
-        packages.hello = pkg system "hello";
-        packages.default = self'.packages.hello;
-        devShells = throw "can't be strict in perSystem.devShells!";
-      };
-      flake.devShells = throw "can't be strict in devShells!";
-    };
-
-  easyOverlay = mkFlake
-    { inputs.self = { }; }
-    {
-      imports = [ flake-parts.flakeModules.easyOverlay ];
-      systems = [ "a" "aarch64-linux" ];
-      perSystem = { system, config, final, pkgs, ... }: {
-        packages.default = config.packages.hello;
-        packages.hello = pkg system "hello";
-        packages.hello_new = final.hello;
-        overlayAttrs = {
-          hello = config.packages.hello;
-          hello_old = pkgs.hello;
-          hello_new = config.packages.hello_new;
-        };
-      };
-    };
-
-  bundlersExample = mkFlake
-    { inputs.self = { }; }
-    {
-      imports = [ flake-parts.flakeModules.bundlers ];
-      systems = [ "a" "b" ];
-      perSystem = { system, ... }: {
-        packages.hello = pkg system "hello";
-        bundlers.toTarball = drv: pkg system "tarball-${drv.name}";
-        bundlers.toAppImage = drv: pkg system "appimage-${drv.name}";
-      };
-    };
-
-  modulesFlake =
-    mkFlake
-      {
-        inputs.self = { };
-        moduleLocation = "modulesFlake";
-      }
-      {
-        imports = [ flake-parts.flakeModules.modules ];
-        options = {
-          # Test option that uses plain types.submodule
-          flake.fooConfiguration = lib.mkOption {
-            type = lib.types.submoduleWith {
-              # Just Like types.submodule;
-              shorthandOnlyDefinesConfig = true;
-              class = "foo";
-              modules = [ ];
-            };
-          };
-        };
-        config = {
-          systems = [ ];
-          flake = {
-            modules.generic.example =
-              { lib, ... }:
-              {
-                options.generic.example = lib.mkOption { default = "works in any module system application"; };
-              };
-            modules.foo.example =
-              { lib, ... }:
-              {
-                options.foo.example = lib.mkOption { default = "works in foo application"; };
-              };
-            fooConfiguration = modulesFlake.modules.foo.example;
-          };
-        };
-      };
-
-  flakeModulesDeclare = mkFlake
-    { inputs.self = { outPath = ./.; }; }
-    ({ config, ... }: {
-      imports = [ flake-parts.flakeModules.flakeModules ];
-      systems = [ ];
-      flake.flakeModules.default = { lib, ... }: {
-        options.flake.test123 = lib.mkOption { default = "option123"; };
-        imports = [ config.flake.flakeModules.extra ];
-      };
-      flake.flakeModules.extra = {
-        flake.test123 = "123test";
-      };
-    });
-
-  flakeModulesImport = mkFlake
-    { inputs.self = { }; }
-    {
-      imports = [ flakeModulesDeclare.flakeModules.default ];
-    };
-
-  flakeModulesDisable = mkFlake
-    { inputs.self = { }; }
-    {
-      imports = [ flakeModulesDeclare.flakeModules.default ];
-      disabledModules = [ flakeModulesDeclare.flakeModules.extra ];
-    };
-
-  nixpkgsWithoutEasyOverlay = import nixpkgs {
-    system = "x86_64-linux";
-    overlays = [ ];
-    config = { };
-  };
-
-  nixpkgsWithEasyOverlay = import nixpkgs {
-    # non-memoized
-    system = "x86_64-linux";
-    overlays = [ easyOverlay.overlays.default ];
-    config = { };
-  };
-
-  nixpkgsWithEasyOverlayMemoized = import nixpkgs {
-    # memoized
-    system = "aarch64-linux";
-    overlays = [ easyOverlay.overlays.default ];
-    config = { };
-  };
-
-  specialArgFlake = mkFlake
-    {
-      inputs.self = { };
-      specialArgs.soSpecial = true;
-    }
-    ({ soSpecial, ... }: {
-      imports = assert soSpecial; [ ];
-      flake.foo = true;
-    });
-
-  partitionWithoutExtraInputsFlake = mkFlake
-    {
-      inputs.self = { };
-    }
-    ({ config, ... }: {
-      imports = [ flake-parts.flakeModules.partitions ];
-      systems = [ "x86_64-linux" ];
-      partitions.dev.module = { inputs, ... }: builtins.seq inputs { };
-      partitionedAttrs.devShells = "dev";
-    });
-
-  nixosModulesFlake = mkFlake
-    {
-      inputs.self = { outPath = "/test/path"; };
-    }
-    {
-      systems = [ ];
-      flake.nixosModules.example = { lib, ... }: {
-        options.test.option = lib.mkOption { default = "nixos-test"; };
-      };
-    };
-
-  /**
-    This one is for manual testing. Should look like:
-
-    ```
-    nix-repl> checks.x86_64-linux.eval-tests.internals.printSystem.withSystem "foo" ({ config, ... }: null)
-    trace: Evaluating perSystem for foo
-    null
-
-    nix-repl> checks.x86_64-linux.eval-tests.internals.printSystem.withSystem "foo" ({ config, ... }: null)
-    null
-
-    ```
-  */
-  printSystem = mkFlake
-    { inputs.self = { }; }
-    ({ withSystem, ... }: {
-      systems = [ ];
-      perSystem = { config, system, ... }:
-        builtins.trace "Evaluating perSystem for ${system}" { };
-      flake.withSystem = withSystem;
-    });
-
-  dogfoodProvider = mkFlake
-    { inputs.self = { }; }
-    ({ flake-parts-lib, ... }: {
-      imports = [
-        (flake-parts-lib.importAndPublish "dogfood" { flake.marker = "dogfood"; })
-      ];
-    });
-
-  dogfoodConsumer = mkFlake
-    { inputs.self = { }; }
-    ({ flake-parts-lib, ... }: {
-      imports = [
-        dogfoodProvider.modules.flake.dogfood
-      ];
-    });
-
-  runTests = ok:
-
-    assert empty == {
-      apps = { };
-      checks = { };
-      devShells = { };
-      formatter = { };
-      legacyPackages = { };
-      nixosConfigurations = { };
-      nixosModules = { };
-      overlays = { };
-      packages = { };
-    };
-
-    assert example1 == {
-      apps = {
-        a = {
-          hello = {
-            program = "${pkg "a" "hello"}/bin/hello";
-            type = "app";
-            meta = { };
-          };
-        };
-        b = {
-          hello = {
-            program = "${pkg "b" "hello"}/bin/hello";
-            type = "app";
-            meta = { };
-          };
-        };
-      };
-      checks = { a = { }; b = { }; };
-      devShells = { a = { }; b = { }; };
-      formatter = { };
-      legacyPackages = { a = { }; b = { }; };
-      nixosConfigurations = { };
-      nixosModules = { };
-      overlays = { };
-      packages = {
-        a = { hello = pkg "a" "hello"; };
-        b = { hello = pkg "b" "hello"; };
-      };
-    };
-
-    assert bundlersExample.bundlers.a.toTarball (pkg "a" "hello") == pkg "a" "tarball-hello";
-    assert bundlersExample.bundlers.b.toAppImage (pkg "b" "hello") == pkg "b" "appimage-hello";
-
-    # - exported package becomes part of overlay.
-    # - perSystem is invoked for the right system, when system is non-memoized
-    assert nixpkgsWithEasyOverlay.hello == pkg "x86_64-linux" "hello";
-
-    # - perSystem is invoked for the right system, when system is memoized
-    assert nixpkgsWithEasyOverlayMemoized.hello == pkg "aarch64-linux" "hello";
-
-    # - Non-exported package does not become part of overlay.
-    assert nixpkgsWithEasyOverlay.default or null != pkg "x86_64-linux" "hello";
-
-    # - hello_old comes from super
-    assert nixpkgsWithEasyOverlay.hello_old == nixpkgsWithoutEasyOverlay.hello;
-
-    # - `hello_new` shows that the `final` wiring works
-    assert nixpkgsWithEasyOverlay.hello_new == nixpkgsWithEasyOverlay.hello;
-
-    assert flakeModulesImport.test123 == "123test";
-
-    assert flakeModulesDisable.test123 == "option123";
-
-    assert packagesNonStrictInDevShells.packages.a.default == pkg "a" "hello";
-
-    assert emptyExposeArgs.moduleLocation == "the self outpath/flake.nix";
-
-    assert (lib.evalModules {
-      class = "barrr";
-      modules = [
-        modulesFlake.modules.generic.example
-      ];
-    }).config.generic.example == "works in any module system application";
-
-    assert (lib.evalModules {
-      class = "foo";
-      modules = [
-        modulesFlake.modules.foo.example
-      ];
-    }).config.foo.example == "works in foo application";
-
-    # Test that modules can be loaded into plain submodules with shorthandOnlyDefinesConfig = true
-    assert modulesFlake.fooConfiguration.foo.example == "works in foo application";
-
-    assert specialArgFlake.foo;
-
-    assert builtins.isAttrs partitionWithoutExtraInputsFlake.devShells.x86_64-linux;
-
-    assert nixosModulesFlake.nixosModules.example._class == "nixos";
-
-    assert nixosModulesFlake.nixosModules.example._file == "/test/path/flake.nix#nixosModules.example";
-
-    assert (lib.evalModules {
-      class = "nixos";
-      modules = [
-        nixosModulesFlake.nixosModules.example
-      ];
-    }).config.test.option == "nixos-test";
-
-    assert dogfoodProvider.marker == "dogfood";
-    assert dogfoodConsumer.marker == "dogfood";
-
-    ok;
-
-  result = runTests "ok";
-}

dev/tests/nix-unit.nix

@@ -0,0 +1,633 @@
+# Tests in nix-unit format
+# Run with:
+#   nix-unit --flake .#tests
+# or:
+#   nix build .#checks.x86_64-linux.nix-unit
+#
+# NOTE: Derivation comparison
+# nix-unit deeply forces both `expr` and `expected` (via forceValueDeep) before
+# comparing. The attrsets returned by `derivation` are self-referential, so
+# this causes stack overflow. We compare `.drvPath` instead where needed, like
+# `==` does when `.type == "derivation"`.
+
+{ flake-parts }:
+let
+  nixpkgs = flake-parts.inputs.nixpkgs;
+  f-p-lib = flake-parts.lib;
+  inherit (f-p-lib) mkFlake;
+  inherit (flake-parts.inputs.nixpkgs-lib) lib;
+
+  pkg = system: name:
+    derivation
+      {
+        name = name;
+        builder = "no-builder";
+        system = system;
+      }
+    // {
+      meta = {
+        mainProgram = name;
+      };
+    };
+
+  # Replace derivations with a simple attrset so nix-unit can compare without
+  # hitting stack overflow on self-referential derivation attrsets.
+  # See NOTE: Derivation comparison
+  canon = v:
+    if v.type or null == "derivation" then { type = "derivation"; inherit (v) drvPath outPath name system; }
+    else if builtins.isAttrs v then builtins.mapAttrs (_: canon) v
+    else v;
+
+  empty = mkFlake
+    { inputs.self = { }; }
+    {
+      systems = [ ];
+    };
+
+  emptyExposeArgs = mkFlake
+    { inputs.self = { outPath = "the self outpath"; }; }
+    ({ config, moduleLocation, ... }: {
+      flake = {
+        inherit moduleLocation;
+      };
+    });
+
+  # Originally tested errorLocation (removed in c8c8e56). Repurposed to test
+  # that mkFlake can produce standard outputs without forcing self.
+  emptyExposeArgsNoSelf = mkFlake
+    { inputs.self = throw "self won't be available in case of some errors"; }
+    {
+      systems = [ ];
+    };
+
+  example1 = mkFlake
+    { inputs.self = { }; }
+    {
+      systems = [ "a" "b" ];
+      perSystem = { config, system, ... }: {
+        packages.hello = pkg system "hello";
+        apps.hello.program = config.packages.hello;
+      };
+    };
+
+  packagesNonStrictInDevShells = mkFlake
+    { inputs.self = packagesNonStrictInDevShells; /* approximation */ }
+    {
+      systems = [ "a" "b" ];
+      perSystem = { system, self', ... }: {
+        packages.hello = pkg system "hello";
+        packages.default = self'.packages.hello;
+        devShells = throw "can't be strict in perSystem.devShells!";
+      };
+      flake.devShells = throw "can't be strict in devShells!";
+    };
+
+  easyOverlay = mkFlake
+    { inputs.self = { }; }
+    {
+      imports = [ flake-parts.flakeModules.easyOverlay ];
+      systems = [ "a" "aarch64-linux" ];
+      perSystem = { system, config, final, pkgs, ... }: {
+        packages.default = config.packages.hello;
+        packages.hello = pkg system "hello";
+        packages.hello_new = final.hello;
+        overlayAttrs = {
+          hello = config.packages.hello;
+          hello_old = pkgs.hello;
+          hello_new = config.packages.hello_new;
+        };
+      };
+    };
+
+  bundlersExample = mkFlake
+    { inputs.self = { }; }
+    {
+      imports = [ flake-parts.flakeModules.bundlers ];
+      systems = [ "a" "b" ];
+      perSystem = { system, ... }: {
+        packages.hello = pkg system "hello";
+        bundlers.toTarball = drv: pkg system "tarball-${drv.name}";
+        bundlers.toAppImage = drv: pkg system "appimage-${drv.name}";
+      };
+    };
+
+  modulesFlake =
+    mkFlake
+      {
+        inputs.self = { };
+        moduleLocation = "modulesFlake";
+      }
+      {
+        imports = [ flake-parts.flakeModules.modules ];
+        options = {
+          # Test option that uses plain types.submodule
+          flake.fooConfiguration = lib.mkOption {
+            type = lib.types.submoduleWith {
+              # Just Like types.submodule;
+              shorthandOnlyDefinesConfig = true;
+              class = "foo";
+              modules = [ ];
+            };
+          };
+        };
+        config = {
+          systems = [ ];
+          flake = {
+            modules.generic.example =
+              { lib, ... }:
+              {
+                options.generic.example = lib.mkOption { default = "works in any module system application"; };
+              };
+            modules.foo.example =
+              { lib, ... }:
+              {
+                options.foo.example = lib.mkOption { default = "works in foo application"; };
+              };
+            fooConfiguration = modulesFlake.modules.foo.example;
+          };
+        };
+      };
+
+  flakeModulesDeclare = mkFlake
+    { inputs.self = { outPath = ./.; }; }
+    ({ config, ... }: {
+      imports = [ flake-parts.flakeModules.flakeModules ];
+      systems = [ ];
+      flake.flakeModules.default = { lib, ... }: {
+        options.flake.test123 = lib.mkOption { default = "option123"; };
+        imports = [ config.flake.flakeModules.extra ];
+      };
+      flake.flakeModules.extra = {
+        flake.test123 = "123test";
+      };
+    });
+
+  flakeModulesImport = mkFlake
+    { inputs.self = { }; }
+    {
+      imports = [ flakeModulesDeclare.flakeModules.default ];
+    };
+
+  flakeModulesDisable = mkFlake
+    { inputs.self = { }; }
+    {
+      imports = [ flakeModulesDeclare.flakeModules.default ];
+      disabledModules = [ flakeModulesDeclare.flakeModules.extra ];
+    };
+
+  nixpkgsWithoutEasyOverlay = import nixpkgs {
+    system = "x86_64-linux";
+    overlays = [ ];
+    config = { };
+  };
+
+  nixpkgsWithEasyOverlay = import nixpkgs {
+    # non-memoized
+    system = "x86_64-linux";
+    overlays = [ easyOverlay.overlays.default ];
+    config = { };
+  };
+
+  nixpkgsWithEasyOverlayMemoized = import nixpkgs {
+    # memoized
+    system = "aarch64-linux";
+    overlays = [ easyOverlay.overlays.default ];
+    config = { };
+  };
+
+  specialArgFlake = mkFlake
+    {
+      inputs.self = { };
+      specialArgs.soSpecial = true;
+    }
+    ({ soSpecial, ... }: {
+      imports = assert soSpecial; [ ];
+      flake.foo = true;
+    });
+
+  partitionWithoutExtraInputsFlake = mkFlake
+    {
+      inputs.self = { };
+    }
+    ({ config, ... }: {
+      imports = [ flake-parts.flakeModules.partitions ];
+      systems = [ "x86_64-linux" ];
+      partitions.dev.module = { inputs, ... }: builtins.seq inputs { };
+      partitionedAttrs.devShells = "dev";
+    });
+
+  nixosModulesFlake = mkFlake
+    {
+      inputs.self = { outPath = "/test/path"; };
+    }
+    {
+      systems = [ ];
+      flake.nixosModules.example = { lib, ... }: {
+        options.test.option = lib.mkOption { default = "nixos-test"; };
+      };
+    };
+
+  dogfoodProvider = mkFlake
+    { inputs.self = { }; }
+    ({ flake-parts-lib, ... }: {
+      imports = [
+        (flake-parts-lib.importAndPublish "dogfood" { flake.marker = "dogfood"; })
+      ];
+    });
+
+  dogfoodConsumer = mkFlake
+    { inputs.self = { }; }
+    ({ flake-parts-lib, ... }: {
+      imports = [
+        dogfoodProvider.modules.flake.dogfood
+      ];
+    });
+
+  withSystemFlake = mkFlake
+    { inputs.self = { }; }
+    ({ withSystem, ... }: {
+      systems = [ "a" "b" ];
+      perSystem = { system, ... }: {
+        packages.hello = pkg system "hello";
+      };
+      flake.withSystem = withSystem;
+    });
+
+in
+{
+  "test: mkFlake does not force self" = {
+    expr = emptyExposeArgsNoSelf;
+    expected = {
+      apps = { };
+      checks = { };
+      devShells = { };
+      formatter = { };
+      legacyPackages = { };
+      nixosConfigurations = { };
+      nixosModules = { };
+      overlays = { };
+      packages = { };
+    };
+  };
+
+  withSystem = {
+    "test: withSystem provides the right system for undeclared system" = {
+      expr = withSystemFlake.withSystem "foo" ({ system, ... }: system);
+      expected = "foo";
+    };
+    "test: withSystem provides perSystem config for undeclared system" = {
+      expr = withSystemFlake.withSystem "foo" ({ config, ... }: config.packages.hello);
+      expected = pkg "foo" "hello";
+    };
+    "test: withSystem provides the right system for declared system" = {
+      expr = withSystemFlake.withSystem "a" ({ system, ... }: system);
+      expected = "a";
+    };
+    "test: withSystem provides perSystem config for declared system" = {
+      expr = withSystemFlake.withSystem "a" ({ config, ... }: config.packages.hello);
+      expected = pkg "a" "hello";
+    };
+  };
+  empty = {
+    "test: empty flake outputs" = {
+      expr = empty;
+      expected = {
+        apps = { };
+        checks = { };
+        devShells = { };
+        formatter = { };
+        legacyPackages = { };
+        nixosConfigurations = { };
+        nixosModules = { };
+        overlays = { };
+        packages = { };
+      };
+    };
+  };
+
+  example1 = {
+    "test: full flake output with two systems" = {
+      expr = example1;
+      expected = {
+        apps = {
+          a = {
+            hello = {
+              program = "${pkg "a" "hello"}/bin/hello";
+              type = "app";
+              meta = { };
+            };
+          };
+          b = {
+            hello = {
+              program = "${pkg "b" "hello"}/bin/hello";
+              type = "app";
+              meta = { };
+            };
+          };
+        };
+        checks = { a = { }; b = { }; };
+        devShells = { a = { }; b = { }; };
+        formatter = { };
+        legacyPackages = { a = { }; b = { }; };
+        nixosConfigurations = { };
+        nixosModules = { };
+        overlays = { };
+        packages = {
+          a = { hello = pkg "a" "hello"; };
+          b = { hello = pkg "b" "hello"; };
+        };
+      };
+    };
+  };
+
+  bundlers = {
+    "test: toTarball bundler" = {
+      expr = bundlersExample.bundlers.a.toTarball (pkg "a" "hello");
+      expected = pkg "a" "tarball-hello";
+    };
+    "test: toAppImage bundler" = {
+      expr = bundlersExample.bundlers.b.toAppImage (pkg "b" "hello");
+      expected = pkg "b" "appimage-hello";
+    };
+  };
+
+  easyOverlay = {
+    "test: exported package in overlay, perSystem invoked for non-memoized system" = {
+      expr = nixpkgsWithEasyOverlay.hello;
+      expected = pkg "x86_64-linux" "hello";
+    };
+
+    "test: exported package in overlay, perSystem invoked for memoized system" = {
+      expr = nixpkgsWithEasyOverlayMemoized.hello;
+      expected = pkg "aarch64-linux" "hello";
+    };
+
+    "test: non-exported package not in overlay" = {
+      expr = nixpkgsWithEasyOverlay.default or null != pkg "x86_64-linux" "hello";
+      expected = true;
+    };
+
+    "test: hello_old comes from super" = {
+      expr = nixpkgsWithEasyOverlay.hello_old.drvPath;
+      # See NOTE: Derivation comparison
+      expected = nixpkgsWithoutEasyOverlay.hello.drvPath;
+    };
+
+    "test: hello_new uses final wiring" = {
+      expr = nixpkgsWithEasyOverlay.hello_new.drvPath;
+      # See NOTE: Derivation comparison
+      expected = nixpkgsWithEasyOverlay.hello.drvPath;
+    };
+  };
+
+  flakeModules = {
+    "test: import flakeModule" = {
+      expr = flakeModulesImport.test123;
+      expected = "123test";
+    };
+
+    "test: disable flakeModule" = {
+      expr = flakeModulesDisable.test123;
+      expected = "option123";
+    };
+  };
+
+  "test: packages not strict in devShells" = {
+    expr = packagesNonStrictInDevShells.packages.a.default;
+    expected = pkg "a" "hello";
+  };
+
+  "test: moduleLocation from self outPath" = {
+    expr = emptyExposeArgs.moduleLocation;
+    expected = "the self outpath/flake.nix";
+  };
+
+  modules = {
+    "test: generic module in arbitrary class" = {
+      expr = (lib.evalModules {
+        class = "barrr";
+        modules = [
+          modulesFlake.modules.generic.example
+        ];
+      }).config.generic.example;
+      expected = "works in any module system application";
+    };
+
+    "test: foo module in foo class" = {
+      expr = (lib.evalModules {
+        class = "foo";
+        modules = [
+          modulesFlake.modules.foo.example
+        ];
+      }).config.foo.example;
+      expected = "works in foo application";
+    };
+
+    "test: modules in submodule with shorthandOnlyDefinesConfig" = {
+      expr = modulesFlake.fooConfiguration.foo.example;
+      expected = "works in foo application";
+    };
+  };
+
+  "test: specialArgs passed to module" = {
+    expr = specialArgFlake.foo;
+    expected = true;
+  };
+
+  "test: partition without extra inputs" = {
+    expr = builtins.isAttrs partitionWithoutExtraInputsFlake.devShells.x86_64-linux;
+    expected = true;
+  };
+
+  nixosModules = {
+    "test: nixosModule has _class nixos" = {
+      expr = nixosModulesFlake.nixosModules.example._class;
+      expected = "nixos";
+    };
+
+    "test: nixosModule has correct _file" = {
+      expr = nixosModulesFlake.nixosModules.example._file;
+      expected = "/test/path/flake.nix#nixosModules.example";
+    };
+
+    "test: nixosModule evaluates correctly" = {
+      expr = (lib.evalModules {
+        class = "nixos";
+        modules = [
+          nixosModulesFlake.nixosModules.example
+        ];
+      }).config.test.option;
+      expected = "nixos-test";
+    };
+  };
+
+  dogfood = {
+    "test: importAndPublish provider" = {
+      expr = dogfoodProvider.marker;
+      expected = "dogfood";
+    };
+
+    "test: importAndPublish consumer" = {
+      expr = dogfoodConsumer.marker;
+      expected = "dogfood";
+    };
+  };
+
+  touchup = {
+    "test: any filter keeps only matching attrs" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            imports = [ flake-parts.flakeModules.touchup ];
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            touchup.any = { attrName, ... }: { enable = attrName == "overlays"; };
+            perSystem = { ... }: {
+              packages.default = throw "packages.default should not be evaluated";
+              packages.hello = throw "packages.hello should not be evaluated";
+            };
+          };
+        in
+        result;
+      expected = {
+        overlays = { };
+      };
+    };
+
+    "test: any with mkDefault, attr override" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            imports = [ flake-parts.flakeModules.touchup ];
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            touchup.any = { ... }: { enable = lib.mkDefault false; };
+            touchup.attr.overlays = { enable = true; };
+            perSystem = { ... }: {
+              packages.default = throw "packages.default should not be evaluated";
+              packages.hello = throw "packages.hello should not be evaluated";
+            };
+          };
+        in
+        result;
+      expected = {
+        overlays = { };
+      };
+    };
+
+    "test: nested attr filtering per system" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            imports = [ flake-parts.flakeModules.touchup ];
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            touchup.attr.packages.attr.aarch64-darwin.attr.bar.enable = false;
+            perSystem = { system, ... }: {
+              packages.foo = pkg system "foo";
+              # This assertion proves the filtered value is never evaluated for darwin
+              packages.bar = assert system == "x86_64-linux"; pkg system "bar";
+            };
+          };
+        in
+        canon result.packages;
+      expected = canon {
+        aarch64-darwin = { foo = pkg "aarch64-darwin" "foo"; };
+        x86_64-linux = { foo = pkg "x86_64-linux" "foo"; bar = pkg "x86_64-linux" "bar"; };
+      };
+    };
+
+    "test: finish and attr composition" = {
+      expr =
+        mkFlake { inputs.self = { }; } {
+          imports = [ flake-parts.flakeModules.touchup ];
+          systems = [ "x86_64-linux" "aarch64-darwin" ];
+          touchup.any = { ... }: { enable = lib.mkDefault false; };
+          touchup.attr.overlays = { enable = true; finish = x: "hoi"; };
+          touchup.finish = x: x // { foo = "bar"; };
+        };
+      expected = {
+        overlays = "hoi";
+        foo = "bar";
+      };
+    };
+
+    # TODO: assert that the error context ("while touching up attribute 'broken'")
+    # appears in the trace. nix-unit's expectedError.msg only matches the thrown
+    # message, not addErrorContext frames.
+    "test: error context when enabled attr throws" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            imports = [ flake-parts.flakeModules.touchup ];
+            systems = [ "x86_64-linux" ];
+            flake.broken = throw "the value is broken";
+          };
+        in
+        result.broken;
+      expectedError.type = "ThrownError";
+      expectedError.msg = "the value is broken";
+    };
+  };
+
+  formatter = {
+    "test: conditional null throws helpful error" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            perSystem = { system, ... }: {
+              formatter =
+                if system == "x86_64-linux" then
+                  derivation { name = "fmt"; builder = "x"; system = system; }
+                else null;
+            };
+          };
+        in
+        result.formatter.aarch64-darwin;
+      expectedError.type = "ThrownError";
+      expectedError.msg = "could not determine statically(.|\n)*touchup\\.attr\\.formatter\\.enable = false";
+    };
+
+    "test: empty when never defined" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            perSystem = { ... }: { };
+          };
+        in
+        result.formatter == { };
+      expected = true;
+    };
+
+    "test: present for all systems" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            perSystem = { system, ... }: {
+              formatter = derivation { name = "fmt"; builder = "x"; inherit system; };
+            };
+          };
+        in
+        result ? formatter && result.formatter ? x86_64-linux && result.formatter ? aarch64-darwin;
+      expected = true;
+    };
+
+    "test: lazy per-system" = {
+      expr =
+        let
+          result = mkFlake { inputs.self = { }; } {
+            systems = [ "x86_64-linux" "aarch64-darwin" ];
+            perSystem = { system, ... }: {
+              formatter =
+                if system == "x86_64-linux" then
+                  derivation { name = "fmt"; builder = "x"; system = system; }
+                else
+                  throw "should not evaluate aarch64-darwin formatter";
+            };
+          };
+        in
+        result.formatter.x86_64-linux.name;
+      expected = "fmt";
+    };
+  };
+}

dev/tests/perSystem-memoize.nix

@@ -0,0 +1,32 @@
+# Test that perSystem evaluation is memoized: calling withSystem twice for the
+# same undeclared system should only evaluate perSystem once.
+# Run with:
+#   nix build .#checks.x86_64-linux.perSystem-memoize
+
+{ flake-parts, runCommand, nix }:
+runCommand "perSystem-memoize" { nativeBuildInputs = [ nix ]; } ''
+  export HOME="$(realpath .)"
+  unset NIX_STORE
+  export NIX_STORE_DIR=${builtins.storeDir}
+  export NIX_REMOTE="$HOME/storedata"
+
+  nix eval \
+    --extra-experimental-features 'nix-command flakes' \
+    --no-write-lock-file \
+    --offline \
+    --substituters "" \
+    --override-input flake-parts ${flake-parts} \
+    --override-input flake-parts/nixpkgs-lib ${flake-parts.inputs.nixpkgs-lib} \
+    "${./perSystem-memoize}#result" \
+    2>eval-stderr
+
+  count=$(grep -c "Evaluating perSystem for foo" eval-stderr || true)
+  cat eval-stderr >&2
+  echo "Trace count: $count"
+  if [ "$count" -ne 1 ]; then
+    echo "FAIL: expected perSystem to be evaluated exactly once, but it was evaluated $count times"
+    exit 1
+  fi
+  echo "PASS: perSystem evaluated exactly once"
+  touch $out
+''

dev/tests/perSystem-memoize/flake.nix

@@ -0,0 +1,15 @@
+{
+  inputs.flake-parts.url = "github:hercules-ci/flake-parts";
+  outputs = inputs:
+    inputs.flake-parts.lib.mkFlake { inherit inputs; } ({ withSystem, ... }: {
+      systems = [ ];
+      perSystem = { system, ... }:
+        builtins.trace "Evaluating perSystem for ${system}" { };
+      flake.result =
+        let
+          a = withSystem "foo" ({ config, ... }: null);
+          b = withSystem "foo" ({ config, ... }: "ok");
+        in
+        builtins.seq a b;
+    });
+}

extras/bundlers.nix

@@ -1,4 +1,4 @@
-# Tests in: ../dev/tests/eval-tests.nix (bundlersExample)
+# Tests in: ../dev/tests/nix-unit.nix (bundlersExample)
 
 { lib
 , flake-parts-lib

extras/touchup.nix

@@ -0,0 +1,57 @@
+{ config, lib, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+in
+{
+  options = {
+    touchup = mkOption {
+      description = ''
+        Controls which attributes appear in [`processedFlake`](flake-parts.md#opt-processedFlake) and how they are transformed.
+
+        The touchup configuration forms a tree that mirrors the flake output structure.
+        At each level, [`attr`](#opt-touchup.attr) targets specific attributes by name,
+        and [`any`](#opt-touchup.any) applies to all attributes at that level.
+
+        **Examples**:
+
+        Only output explicitly listed flake output attributes:
+
+        ```nix
+        touchup = {
+          any = {
+            enable = lib.mkDefault false;
+          };
+          attr.packages.enable = true;
+          attr.checks.enable = true;
+        }
+        ```
+
+        Hide a package from users, but not from your own modules:
+
+        ```nix
+        touchup = {
+          attr.packages.any.attr.hello.enable = false;
+        };
+        ```
+
+        Hide a package on a set of systems:
+
+        ```nix
+        touchup = {
+          attr.packages.any = { attrName, ... }: { attr.hello.enable = ! lib.strings.hasSuffix "-darwin" attrName; }
+        };
+        ```
+
+      '';
+      type = types.submoduleWith {
+        modules = [ ./touchup/attrs.nix ];
+      };
+    };
+  };
+  config = {
+    processedFlake = config.touchup.touchupApply config.flake;
+  };
+}

extras/touchup/attr.nix

@@ -0,0 +1,20 @@
+{ lib, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+in
+{
+  imports = [ ./attrs.nix ];
+  options = {
+    enable = mkOption {
+      type = types.bool;
+      default = true;
+      description = "Whether to include the attribute in the output.";
+    };
+  };
+  config = {
+    _module.args.docsVisible = "shallow";
+  };
+}

extras/touchup/attrs.nix

@@ -0,0 +1,131 @@
+{ config, lib, options, docsVisible, ... }:
+let
+  inherit (lib)
+    addErrorContext
+    concatMapAttrs
+    evalModules
+    mkOption
+    types
+    ;
+
+  unNull = v:
+    if v == null then
+      { }
+    else
+      v;
+
+  touchupAttrs =
+    v:
+    concatMapAttrs
+      (name: value:
+      let
+        eval = evalModules {
+          prefix = lib.lists.init options.attr.loc; # arbitrary pick
+          specialArgs = {
+            attrName = name;
+          };
+          modules = [
+            (config.attr.${name} or ./attr.nix)
+            (unNull config.any)
+          ];
+        };
+      in
+      if addErrorContext "while figuring out whether to enable '${name}'" eval.config.enable then
+      # Apply the touchup configuration to the value.
+        {
+          "${name}" =
+            addErrorContext "while touching up attribute '${name}'" (
+              (addErrorContext "while evaluating the touchup configuration for '${name}'" eval.config.touchupApply)
+                (addErrorContext "while evaluating the original value of '${name}'" value)
+            );
+        }
+      else
+        { }
+      )
+      v;
+in
+{
+  options = {
+    attr = mkOption {
+      type = types.lazyAttrsOf (types.deferredModuleWith {
+        staticModules = [ ./attr.nix ];
+      });
+      default = { };
+      visible = docsVisible;
+      description =
+        if docsVisible == "shallow" then ''
+          Touchup configuration for the next level of nesting.
+          Same structure as [`attr`](#opt-touchup.attr); see its description.
+        ''
+        else ''
+          Per-attribute touchup configuration. Each value is a module that controls
+          whether and how the corresponding attribute appears in the output.
+
+          Each module contains the full touchup option set (`enable`, `attr`, `any`, `finish`),
+          so nested attributes can be configured to arbitrary depth.
+
+          This module is called with module argument `attrName`, which is the name of the attribute being touched up.
+        '';
+    };
+    any = mkOption {
+      type = types.nullOr (types.deferredModuleWith {
+        staticModules = [ ./attr.nix ];
+      });
+      default = null;
+      visible = docsVisible;
+      description =
+        if docsVisible == "shallow" then ''
+          Default configuration for all attributes at the next level.
+          Same structure as [`any`](#opt-touchup.any); see its description.
+        ''
+        else ''
+          A module whose options are merged into every attribute's touchup configuration.
+          For example, `any.enable = false` disables all attributes by default.
+          Override specific ones via `attr.<name>`.
+
+          Only applies to immediate children — does not recurse into nested attributes automatically.
+
+          This module is called with module argument `attrName`, which is the name of the attribute being touched up.
+        '';
+    };
+
+    type = mkOption {
+      type = types.raw;
+      default = types.raw;
+      defaultText = "raw";
+      description = ''
+        The type used for merging multiple definitions of ${options.finish}.
+        Override this if multiple modules need to compose their ${options.finish} functions.
+      '';
+    };
+    finish = mkOption {
+      type = types.functionTo config.type;
+      default = v: v;
+      defaultText = lib.literalMD "`v: v`, the identity function";
+      description = ''
+        A function applied after filtering and transforming (e.g. by ${options.attr} and ${options.any} at this level).
+        It receives the resulting attribute set and must return the value to use in its place.
+      '';
+    };
+
+    touchupApply = mkOption {
+      internal = true;
+      description = ''
+        A generated function that applies the touchups that are configured with the other options in this module.
+      '';
+      type = types.functionTo types.raw;
+      readOnly = true;
+    };
+  };
+  config = {
+    _module.args.docsVisible = lib.mkDefault true;
+    touchupApply = v:
+      config.finish
+        (if config.attr != { } || config.any != null
+        then
+          addErrorContext "while applying touchups from ${options.attr}: ${lib.options.showDefs options.attr.definitionsWithLocations}\n  and from ${options.any}: ${lib.options.showDefs options.any.definitionsWithLocations}"
+            (touchupAttrs v)
+        else v);
+  };
+}
+

flake.nix

@@ -51,6 +51,7 @@
         modules = ./extras/modules.nix;
         partitions = ./extras/partitions.nix;
         bundlers = ./extras/bundlers.nix;
+        touchup = ./extras/touchup.nix;
       };
     in
     lib.mkFlake { inherit inputs; } {
@@ -59,6 +60,7 @@
       partitionedAttrs.checks = "dev";
       partitionedAttrs.devShells = "dev";
       partitionedAttrs.herculesCI = "dev";
+      partitionedAttrs.tests = "dev";
       partitions.dev.extraInputsFlake = ./dev;
       partitions.dev.module = {
         imports = [ ./dev/flake-module.nix ];

lib.nix

@@ -159,7 +159,7 @@ let
       let
         eval = flake-parts-lib.evalFlakeModule args module;
       in
-      eval.config.flake;
+      eval.config.processedFlake;
 
     /**
       Deprecated. Declare options directly, e.g. `options.foo.bar = mkOption { ... }`,

modules/flake.nix

@@ -1,4 +1,4 @@
-{ lib, ... }:
+{ config, lib, options, ... }:
 let
   inherit (lib)
     mkOption
@@ -33,6 +33,27 @@ in
         Raw flake output attributes. Any attribute can be set here, but some
         attributes are represented by options, to provide appropriate
         configuration merging.
+
+        Further processing may be applied to these attributes. See [`processedFlake`](flake-parts.md#opt-processedFlake) for more information.
+      '';
+    };
+    processedFlake = mkOption {
+      type = types.raw;
+      readOnly = true;
+      apply =
+        if options.processedFlake.isDefined
+        then x: x
+        else x: config.flake;
+      description = ''
+        The final flake output, as returned by `mkFlake`.
+
+        This separates two concerns:
+
+        - [`flake`](#opt-flake): the internal representation, where all attributes are available for use by other modules, regardless of whether they evaluate successfully for all configurations.
+        - `processedFlake`: the external output, which may have attributes removed or transformed to satisfy tools like `nix flake check`.
+
+        By default, `processedFlake` equals `flake` (no processing).
+        Import a module such as [`flakeModules.touchup`](#opt-touchup) to define it, or set it directly.
       '';
     };
   };

modules/formatter.nix

@@ -1,7 +1,6 @@
 { config, lib, flake-parts-lib, ... }:
 let
   inherit (lib)
-    filterAttrs
     mapAttrs
     mkOption
     optionalAttrs
@@ -10,11 +9,41 @@ let
   inherit (flake-parts-lib)
     mkPerSystemOption
     ;
+
+  # Do not copy this pattern! (probe, haveFormatterProbably, optionalAttrs)
+  # It kind of works somewhat for `formatter`, but it is bad.
+  # Nothing critical must rely on this!
+  # - `tryEval` makes debugging harder and more annoying
+  # - There's a performance cost to this otherwise useless evaluation
+  # - We only use it *in `formatter`* because the effects are limited to that
+  #   particular attribute. This makes the risk somewhat manageable.
+  probe =
+    config.perSystem
+      # Elaborate error message that users should never see.
+      # The reason to even do this, is that we can sidestep
+      # https://github.com/hercules-ci/flake-parts/issues/288 in many cases,
+      # without flake author intervention to keep `nix flake check` happy.
+      (throw ''
+        For the purpose of finding out whether an option may be unset for all systems, flake-parts probes the perSystem module without a valid `system` argument, and tries to catch this exception if it finds that `system` is required for this determination. If you see this message, it means that for some reason, flake-parts was unable to catch this exception.
+
+        This may be a bug in Nix or in flake-parts, but ultimately this is due to the quirky requirement of flakes that the "system" attribute does not come first.
+        Flake-parts tries its best to correct that UX, but ultimately, this needs to be solved in Nix.
+      '');
+
+  haveFormatterProbably =
+    let
+      ev =
+        builtins.tryEval
+          probe.formatter;
+    in
+    # If it fails, we can't assume that we don't have a formatter, because it may well evaluate for a real system.
+    !ev.success
+    || ev.value != null;
 in
 {
   options = {
     flake.formatter = mkOption {
-      type = types.lazyAttrsOf types.package;
+      type = types.lazyAttrsOf (types.nullOr types.package);
       default = { };
       description = ''
         An attribute set of per system a package used by [`nix fmt`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-fmt.html).
@@ -36,12 +65,40 @@ in
   };
   config = {
     flake.formatter =
-      mapAttrs
+      # Work around `nix flake check` not allowing `null` values in output attributes.
-        (k: v: v.formatter)
+      optionalAttrs
-        (filterAttrs
+        haveFormatterProbably
-          (k: v: v.formatter != null)
+        (mapAttrs
-          config.allSystems
+          (k: v:
-        );
+            if v.formatter != null then v.formatter
+            else
+              throw ''
+                flake-parts could not determine statically that no formatter is defined for *all* systems.
+
+                What happened?
+
+                1. For performance reasons, flake-parts must not query `perSystem` for every system, so it uses a heuristic to determine whether a formatter is defined for all systems.
+                2. Unfortunately, this heuristic is not perfect, and it wasn't able to determine for your flake that `perSystem.formatter` is always `null` (if it even is always `null`).
+                As a consequence of (1), flake-parts had to provide the output attribute `formatter.${k}`, but as a consequence of (2), you're seeing this error.
+
+                What to do?
+
+                This whole situation should be temporary. `nix flake check`/`show` can be changed to allow `null` values, which gives flake-parts and other frameworks a way to avoid this situation.
+
+                To change the `formatter` output attribute, you can control it precisely with the `touchup` module, for example:
+
+                    imports = [ inputs.flake-parts.flakeModules.touchup ];
+
+                    # Remove it
+                    touchup.attr.formatter.enable = false;
+
+                    # ... or only have it for listed systems
+                    touchup.attr.formatter.any.enable = lib.mkDefault false;
+                    touchup.attr.formatter.attr.x86_64-linux.enable = true;
+                    touchup.attr.formatter.attr.aarch64-darwin.enable = true;
+
+              '')
+          config.allSystems);
 
     perInput = system: flake:
       optionalAttrs (flake?formatter.${system}) {