nix/11.stylix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ci: run all builds in a single job (#1069)

Browse source 
This works around GitHub's limit on the number of matrix jobs (fixes
#947), by running all builds in a single job. To maintain some speed, we
use `nix-fast-build`, which uses multiple cores for evaluation, and skips
any builds for which the final derivation is already in a binary cache.

Although this makes the run for an individual pull request slower, the
amount of duplicated work is greatly reduced: previously, we often had
100 machines building the same derivation in parallel. This means that
more runners are available should there be multiple pull requests opened
in a short space of time, so there is less queuing. It's also more
energy efficient.

A potential downside is that the logs are all merged together, so it can
be hard to find what failed when lots of outputs were built.
`nix-fast-build` does report a list of failed attributes at the end of
the log, but this is currently broken:
https://github.com/Mic92/nix-fast-build/pull/98

The script used to launch `nix-fast-build` is also added to the
developer shell for local use. This replaces the old `nix-flake-check`
package (closes #898).

I also saw the opportunity to enable checks on `aarch64-linux` and
`aarch64-darwin` - as these are available as GitHub hosted runners.
This commit is contained in:
Daniel Thwaites 2025-03-30 12:26:07 +01:00 • committed by GitHub
parent 8fce91704d
commit 20117a58eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 91 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

98
.github/workflows/check.yml vendored
View file

@ -12,83 +12,27 @@ permissions:
contents: read
jobs:
get-derivations:
runs-on: ubuntu-24.04
steps:
- uses: DeterminateSystems/nix-installer-action@v16
- uses: cachix/cachix-action@v16
with:
name: stylix
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
continue-on-error: true
- id: get-derivations
run: |
set -o pipefail
nix flake show --json \
github:${{
github.repository
}}/${{
github.event.pull_request.head.sha || github.sha
}} |
jq --raw-output '
def format_output($arch; $type):
{
arch: $arch,
key: .,
os: (
if $arch == "x86_64-linux" then
"ubuntu-24.04"
else
"macos-14"
end
),
type: $type
};
[
["x86_64-linux", "x86_64-darwin"][] as $arch |
(.checks[$arch] | keys) as $checks |
(.packages[$arch] | keys) as $packages |
(($checks - $packages)[] | format_output($arch; "checks")),
($packages[] | format_output($arch; "packages"))
] |
"derivations=\(.)"
' \
>>"$GITHUB_OUTPUT" || {
rm "$GITHUB_OUTPUT"
false
}
outputs:
derivations: ${{ steps.get-derivations.outputs.derivations }}
check:
runs-on: ${{ matrix.check.os }}
name: ${{ matrix.check.key }} on ${{ matrix.check.arch }}
needs: get-derivations
name: ${{ matrix.name }}
runs-on: ${{ matrix.runs-on }}
# https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/choosing-the-runner-for-a-job#choosing-github-hosted-runners
strategy:
fail-fast: false
matrix:
check: ${{ fromJSON(needs.get-derivations.outputs.derivations) }}
include:
- name: aarch64-linux
runs-on: ubuntu-24.04-arm
- name: aarch64-darwin
runs-on: macos-15
- name: x86_64-linux
runs-on: ubuntu-24.04
- name: x86_64-darwin
runs-on: macos-13
steps:
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@v16
with:
extra-conf: |-
allow-import-from-derivation = ${{
startsWith(matrix.check.key, 'testbed:') &&
contains(matrix.check.key, ':schemeless') &&
'true' ||
'false'
}}
- uses: cachix/cachix-action@v16
with:
@ -96,16 +40,4 @@ jobs:
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
continue-on-error: true
- run: |
nix build --no-update-lock-file --print-build-logs \
github:${{
github.repository
}}/${{
github.event.pull_request.head.sha || github.sha
}}#${{
matrix.check.type
}}.${{
matrix.check.arch
}}.${{
matrix.check.key
}}
- run: nix develop --command stylix-check --no-nom

48
docs/src/development_environment.md
View file

@ -4,34 +4,56 @@
To enter the developer shell, run:
```console
```sh
nix develop
```
To automatically enter the developer shell upon entering the project directory
with [`direnv`](https://direnv.net), run:
```console
```sh
direnv allow
```
## pre-commit
## `pre-commit`
The default developer shell leverages [`pre-commit`](https://pre-commit.com)
hooks to simplify the process of reaching minimum quality standards for casual
contributors.
contributors. This means applying code formatters, and scanning for things like
unused variables which should be removed.
By default, `pre-commit` only runs on staged files. To manually run
[`pre-commit`](https://pre-commit.com) against all files, run:
By default, once you have entered the developer shell, `pre-commit` runs
automatically just before you create a commit. This will only look at the
files which are about to be committed.
```console
You can also run it manually against all files:
```sh
pre-commit run --all-files
```
This is useful when submitting a patchset and `pre-commit` was not used on all
commits. For example, suppose the first commit was created without `pre-commit`
and touches `/flake.nix`. Installing `pre-commit` and then creating a second
commit that touches `/README.md` will not run any hooks on `/flake.nix`.
This is useful if a commit was created outside of the developer shell, and
you need to apply `pre-commit` to your previous changes.
Note that the `outputs.checks.${system}.git-hooks` output always runs against
all files.
Note that there is also a flake output, `.#checks.«system».git-hooks`, which
always runs against all files but does not have access to apply changes. This
is used in GitHub Actions to ensure that `pre-commit` has been applied.
## `stylix-check`
When a pull request is opened, we use GitHub Actions to build everything under
`.#checks`. This includes the previously mentioned `.#checks.«system».git-hooks`,
and every [testbed](./testbeds.md).
You might sometimes find it useful to run these same checks locally. The built
in `nix flake check` command does this, however it can be quite slow compared
to the script we use on GitHub Actions.
To use the same script that we use, you can run this command within the
developer shell:
```sh
stylix-check
```
This is based on [`nix-fast-build`](https://github.com/Mic92/nix-fast-build#readme).

81
flake.nix
View file

@ -153,14 +153,36 @@
} self.packages.${system};
devShells = {
default = pkgs.mkShell {
inherit (self.checks.${system}.git-hooks) shellHook;
default =
let
check = pkgs.writeShellApplication {
name = "stylix-check";
runtimeInputs = with pkgs; [
nix
nix-fast-build
];
text = ''
cores="$(nproc)"
system="$(nix eval --expr builtins.currentSystem --impure --raw)"
nix-fast-build \
--eval-max-memory-size 512 \
--eval-workers "$cores" \
--flake ".#checks.$system" \
--no-link \
--skip-cached \
"$@"
'';
};
in
pkgs.mkShell {
inherit (self.checks.${system}.git-hooks) shellHook;
packages = [
inputs.home-manager.packages.${system}.default
self.checks.${system}.git-hooks.enabledPackages
];
};
packages = [
check
inputs.home-manager.packages.${system}.default
self.checks.${system}.git-hooks.enabledPackages
];
};
ghc = pkgs.mkShell {
inputsFrom = [ self.devShells.${system}.default ];
@ -172,38 +194,6 @@
let
universalPackages = {
docs = import ./docs { inherit pkgs inputs lib; };
nix-flake-check = pkgs.writeShellApplication {
meta.description = "A parallelized alternative to 'nix flake check'";
name = "nix-flake-check";
runtimeInputs = with pkgs; [
nix
jq
parallel
];
text = ''
nix flake show --json --no-update-lock-file ${self} |
jq --raw-output '
((.checks."${system}" // {}) | keys) as $checks |
((.packages."${system}" // {}) | keys) as $packages |
(($checks - $packages)[] | "checks.${system}.\(.)"),
($packages[] | "packages.${system}.\(.)")
' |
parallel \
--bar \
--color \
--color-failed \
--halt now,fail=1 \
--tagstring '{}' \
'
nix build --no-update-lock-file --print-build-logs \
${self}#{}
'
'';
};
palette-generator = pkgs.callPackage ./palette-generator { };
};
@ -212,8 +202,19 @@
testbedPackages = lib.optionalAttrs (lib.hasSuffix "-linux" system) (
import ./stylix/testbed.nix { inherit pkgs inputs lib; }
);
# Discord is not available on arm64. This workaround filters out
# testbeds using that package, until we have a better way to handle
# this.
testbedPackages' =
if system == "aarch64-linux" then
lib.filterAttrs (
name: _: !lib.hasPrefix "testbed:discord:vencord" name
) testbedPackages
else
testbedPackages;
in
universalPackages // testbedPackages;
universalPackages // testbedPackages';
}
)
// {