diff --git a/docs/eval-bench.sh b/docs/eval-bench.sh new file mode 100755 index 0000000..8c4e70c --- /dev/null +++ b/docs/eval-bench.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Ralph eval-time benchmark. +# Usage: ralph-bench.sh +# Primary metric: median cpuTime (NIX_SHOW_STATS) — stable, isolates eval work. +# Secondary: median wall-clock (noisy, IO-bound). +# Eval cache disabled; fetcher cache warm via persistent $HOME. +set -uo pipefail + +RUNS="${1:-5}" +export HOME=/tmp/ralph-evalhome +mkdir -p "$HOME" +FLAKE="${FLAKE:-/home/srid/code/nixos-config/.worktrees/ralph}" + +declare -A ATTR +ATTR[pureintent]="$FLAKE#nixosConfigurations.pureintent.config.system.build.toplevel.drvPath" +ATTR[zest]="$FLAKE#homeConfigurations.\"srid@zest\".activationPackage.drvPath" + +median() { sort -n | awk '{a[NR]=$1} END{ if(NR%2){print a[(NR+1)/2]} else {print (a[NR/2]+a[NR/2+1])/2} }'; } +now_ms() { date +%s%3N; } + +for target in pureintent zest; do + attr="${ATTR[$target]}" + cpus=(); walls=() + for i in $(seq 1 "$RUNS"); do + start=$(now_ms) + NIX_SHOW_STATS=1 nix eval --raw "$attr" --option eval-cache false \ + >/dev/null 2>/tmp/ralph-stats.json + rc=$? + end=$(now_ms) + if [ $rc -ne 0 ]; then echo "$target run $i FAILED (rc=$rc)"; tail -8 /tmp/ralph-stats.json; exit 1; fi + cpu=$(grep -o '"cpuTime": [0-9.]*' /tmp/ralph-stats.json | head -1 | grep -o '[0-9.]*') + cpus+=("$cpu"); walls+=("$((end - start))") + done + medcpu=$(printf '%s\n' "${cpus[@]}" | median) + medwall=$(printf '%s\n' "${walls[@]}" | median) + medwall_s=$(awk "BEGIN{printf \"%.2f\", $medwall/1000}") + printf '%-11s cpuTime=%ss (median of %d) | wall=%ss | cpu=[%s] wall_ms=[%s]\n' \ + "$target" "$medcpu" "$RUNS" "$medwall_s" "${cpus[*]}" "${walls[*]}" +done diff --git a/docs/eval-time-ralph-report.md b/docs/eval-time-ralph-report.md new file mode 100644 index 0000000..8271172 --- /dev/null +++ b/docs/eval-time-ralph-report.md @@ -0,0 +1,54 @@ +# Nix Eval-Time Optimization (Ralph) + +Iterative measurement-driven reduction of Nix **evaluation** time for the two +configs that get evaluated most often on this machine: + +- `pureintent` — the NixOS system (`nixosConfigurations.pureintent`) +- `zest` — the home-manager config (`homeConfigurations."srid@zest"`) + +## Methodology + +Primary metric: **`cpuTime`** from `NIX_SHOW_STATS=1` — it isolates pure +evaluation work and is stable run-to-run (~3% spread). Wall-clock is reported +too but is IO/cache-bound and noisy (±15%), so it is *not* used for commit +decisions. + +```sh +# docs/eval-bench.sh (default 5) +# - persistent $HOME=/tmp/ralph-evalhome keeps the fetcher/git cache warm +# - --option eval-cache false forces a full re-eval every run +# - reports median cpuTime + median wall over N runs, per target +nix eval --raw .#nixosConfigurations.pureintent.config.system.build.toplevel.drvPath --option eval-cache false +nix eval --raw '.#homeConfigurations."srid@zest".activationPackage.drvPath' --option eval-cache false +``` + +No `nix store gc` between runs (per request). The eval cache is disabled rather +than cleared, so the warm fetcher cache removes input-fetch noise. + +**Commit rule:** commit only if median cpuTime improves >3% for a target with no +regression on the other. Behaviour may change *only* by dropping inputs/features +first confirmed unused. + +## Baseline (median of 5) + +| Target | cpuTime (s) | wall (s) | cpuTime runs | +|------------|-------------|----------|--------------| +| pureintent | **23.17** | 38.91 | 21.99 / 23.45 / 23.76 / 23.17 / 22.74 | +| zest | **14.93** | 22.09 | 14.93 / 14.50 / 13.86 / 15.49 / 14.96 | + +Env: nix 2.34.7, x86_64-linux. Stats at baseline (pureintent): nrThunks 20.4M, +nrFunctionCalls 13.0M, sets.bytes 868MB, values.bytes 503MB, gcFraction 0.058. + +## Optimization log + +| Cycle | Change | pureintent | zest | Verdict | +|-------|--------|------------|------|---------| +| 0 | baseline | 23.17 | 14.93 | — | + +## Dead ends + +_(investigated, no improvement — recorded so we don't retry)_ + +## Key findings + +_(filled in at wrap-up)_