From 333a5cd066a1a580d8199da222b12815053a3562 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 19:15:22 -0800 Subject: [PATCH 1/7] Async nur commands --- ci/nur/__init__.py | 3 ++- ci/nur/combine.py | 2 +- ci/nur/eval.py | 2 +- ci/nur/format_manifest.py | 2 +- ci/nur/index.py | 2 +- ci/nur/prefetch.py | 56 ++++++++++++++++++++++++++------------- ci/nur/update.py | 50 +++++++++++----------------------- 7 files changed, 58 insertions(+), 59 deletions(-) diff --git a/ci/nur/__init__.py b/ci/nur/__init__.py index b9ba402ae..55dddf215 100644 --- a/ci/nur/__init__.py +++ b/ci/nur/__init__.py @@ -1,6 +1,7 @@ import argparse import logging import sys +import asyncio from typing import List from .combine import combine_command @@ -60,4 +61,4 @@ def main() -> None: args = parse_arguments(sys.argv) logging.basicConfig(level=LOG_LEVELS[args.log_level]) - args.func(args) + asyncio.run(args.func(args)) diff --git a/ci/nur/combine.py b/ci/nur/combine.py index 9ca754f93..07d77b479 100644 --- a/ci/nur/combine.py +++ b/ci/nur/combine.py @@ -174,7 +174,7 @@ def setup_combined() -> None: commit_files(vcs_files, "update code") -def combine_command(args: Namespace) -> None: +async def combine_command(args: Namespace) -> None: combined_path = Path(args.directory) with chdir(combined_path): diff --git a/ci/nur/eval.py b/ci/nur/eval.py index b74a41386..69963526f 100644 --- a/ci/nur/eval.py +++ b/ci/nur/eval.py @@ -60,7 +60,7 @@ import {EVALREPO_PATH} {{ raise EvalError(f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}") -def eval_command(args: Namespace) -> None: +async def eval_command(args: Namespace) -> None: logging.basicConfig(level=logging.INFO) repo_path = Path(args.directory) diff --git a/ci/nur/format_manifest.py b/ci/nur/format_manifest.py index bc397c25f..e35c1d76e 100644 --- a/ci/nur/format_manifest.py +++ b/ci/nur/format_manifest.py @@ -6,7 +6,7 @@ from argparse import Namespace from .path import ROOT -def format_manifest_command(args: Namespace) -> None: +async def format_manifest_command(args: Namespace) -> None: path = ROOT.joinpath("repos.json") manifest = json.load(open(path)) for name, repo in manifest.get("repos", []).items(): diff --git a/ci/nur/index.py b/ci/nur/index.py index 496229993..434725b8a 100644 --- a/ci/nur/index.py +++ b/ci/nur/index.py @@ -94,7 +94,7 @@ callPackage (nur.repo-sources."%s" + "/%s") {} return pkgs -def index_command(args: Namespace) -> None: +async def index_command(args: Namespace) -> None: directory = Path(args.directory) manifest_path = directory.joinpath("repos.json") with open(manifest_path) as f: diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index 31313d791..b3d7341bc 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -2,21 +2,28 @@ import json import os import re import subprocess +import asyncio from pathlib import Path from typing import Optional, Tuple -from urllib.parse import ParseResult +from urllib.parse import urlparse, ParseResult from .error import NurError from .manifest import LockedVersion, Repo, RepoType Url = ParseResult - -def nix_prefetch_zip(url: str) -> Tuple[str, Path]: - data = subprocess.check_output( - ["nix-prefetch-url", "--name", "source", "--unpack", "--print-path", url] +async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: + proc = await asyncio.create_subprocess_exec( + *["nix-prefetch-url", "--name", "source", "--unpack", "--print-path", url], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) - sha256, path = data.decode().strip().split("\n") + stdout, stderr = await proc.communicate() + + if proc.returncode != 0: + raise RuntimeError(stderr.decode()) + + sha256, path = stdout.decode().strip().split("\n") return sha256, Path(path) @@ -24,21 +31,32 @@ class GitPrefetcher: def __init__(self, repo: Repo) -> None: self.repo = repo - def latest_commit(self) -> str: - data = subprocess.check_output( - ["git", "ls-remote", self.repo.url.geturl(), self.repo.branch or "HEAD"], + async def latest_commit(self) -> str: + proc = await asyncio.create_subprocess_exec( + *["git", "ls-remote", self.repo.url.geturl(), self.repo.branch or "HEAD"], env={**os.environ, "GIT_ASKPASS": "", "GIT_TERMINAL_PROMPT": "0"}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) - return data.decode().split(maxsplit=1)[0] + stdout, stderr = await proc.communicate() - def prefetch(self, ref: str) -> Tuple[str, Path]: + if proc.returncode != 0: + raise RuntimeError(stderr.decode()) + + return stdout.decode().split(maxsplit=1)[0] + + async def prefetch(self, ref: str) -> Tuple[str, Path]: cmd = ["nix-prefetch-git"] if self.repo.submodules: cmd += ["--fetch-submodules"] if self.repo.branch: cmd += ["--rev", f"refs/heads/{self.repo.branch}"] cmd += [self.repo.url.geturl()] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) try: stdout, stderr = proc.communicate(timeout=30) except subprocess.TimeoutExpired: @@ -62,12 +80,12 @@ class GitPrefetcher: class GithubPrefetcher(GitPrefetcher): - def prefetch(self, ref: str) -> Tuple[str, Path]: - return nix_prefetch_zip(f"{self.repo.url.geturl()}/archive/{ref}.tar.gz") + async def prefetch(self, ref: str) -> Tuple[str, Path]: + return await nix_prefetch_zip(f"{self.repo.url.geturl()}/archive/{ref}.tar.gz") class GitlabPrefetcher(GitPrefetcher): - def prefetch(self, ref: str) -> Tuple[str, Path]: + async def prefetch(self, ref: str) -> Tuple[str, Path]: hostname = self.repo.url.hostname assert ( hostname is not None @@ -75,10 +93,10 @@ class GitlabPrefetcher(GitPrefetcher): path = Path(self.repo.url.path) escaped_path = "%2F".join(path.parts[1:]) url = f"https://{hostname}/api/v4/projects/{escaped_path}/repository/archive.tar.gz?sha={ref}" - return nix_prefetch_zip(url) + return await nix_prefetch_zip(url) -def prefetch(repo: Repo) -> Tuple[Repo, LockedVersion, Optional[Path]]: +async def prefetch(repo: Repo) -> Tuple[Repo, LockedVersion, Optional[Path]]: prefetcher: GitPrefetcher if repo.type == RepoType.GITHUB: prefetcher = GithubPrefetcher(repo) @@ -87,11 +105,11 @@ def prefetch(repo: Repo) -> Tuple[Repo, LockedVersion, Optional[Path]]: else: prefetcher = GitPrefetcher(repo) - commit = prefetcher.latest_commit() + commit = await prefetcher.latest_commit() locked_version = repo.locked_version if locked_version is not None: if locked_version.rev == commit: return repo, locked_version, None - sha256, path = prefetcher.prefetch(commit) + sha256, path = await prefetcher.prefetch(commit) return repo, LockedVersion(repo.url, commit, sha256, repo.submodules), path diff --git a/ci/nur/update.py b/ci/nur/update.py index 4834b3aec..f30835b43 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -10,8 +10,8 @@ from .prefetch import prefetch logger = logging.getLogger(__name__) -def update(repo: Repo) -> Repo: - repo, locked_version, repo_path = prefetch(repo) +async def update(repo: Repo) -> Repo: + repo, locked_version, repo_path = await prefetch(repo) if repo_path: eval_repo(repo, repo_path) @@ -20,42 +20,22 @@ def update(repo: Repo) -> Repo: return repo -def update_command(args: Namespace) -> None: +async def update_command(args: Namespace) -> None: logging.basicConfig(level=logging.INFO) manifest = load_manifest(MANIFEST_PATH, LOCK_PATH) - if getattr(args, "debug", False): - for repo in manifest.repos: - try: - update(repo) - except EvalError as err: - if repo.locked_version is None: - logger.error( - f"repository {repo.name} failed to evaluate: {err}. This repo is not yet in our lock file!!!!" - ) - raise - logger.error(f"repository {repo.name} failed to evaluate: {err}") - except Exception: - logger.exception(f"Failed to update repository {repo.name}") - else: - with ThreadPoolExecutor() as executor: - future_to_repo = { - executor.submit(update, repo): repo for repo in manifest.repos - } - - for future in as_completed(future_to_repo): - repo = future_to_repo[future] - try: - future.result() - except EvalError as err: - if repo.locked_version is None: - logger.error( - f"repository {repo.name} failed to evaluate: {err}. This repo is not yet in our lock file!!!!" - ) - raise - logger.error(f"repository {repo.name} failed to evaluate: {err}") - except Exception: - logger.exception(f"Failed to update repository {repo.name}") + for repo in manifest.repos: + try: + await update(repo) + except EvalError as err: + if repo.locked_version is None: + logger.error( + f"repository {repo.name} failed to evaluate: {err}. This repo is not yet in our lock file!!!!" + ) + raise + logger.error(f"repository {repo.name} failed to evaluate: {err}") + except Exception: + logger.exception(f"Failed to update repository {repo.name}") update_lock_file(manifest.repos, LOCK_PATH) From 3692fecb548cc562abb1c867d2b4b39947a5ed59 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 19:49:36 -0800 Subject: [PATCH 2/7] Some prefetch bug fixes --- ci/nur/prefetch.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index b3d7341bc..fa56735bc 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -21,7 +21,9 @@ async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: stdout, stderr = await proc.communicate() if proc.returncode != 0: - raise RuntimeError(stderr.decode()) + raise NurError( + f"Failed to prefetch git repository {url}: {stderr.decode()}" + ) sha256, path = stdout.decode().strip().split("\n") return sha256, Path(path) @@ -41,7 +43,9 @@ class GitPrefetcher: stdout, stderr = await proc.communicate() if proc.returncode != 0: - raise RuntimeError(stderr.decode()) + raise NurError( + f"Failed to prefetch git repository {self.repo.url.geturl()}: {stderr.decode()}" + ) return stdout.decode().split(maxsplit=1)[0] @@ -58,7 +62,7 @@ class GitPrefetcher: stderr=asyncio.subprocess.PIPE, ) try: - stdout, stderr = proc.communicate(timeout=30) + stdout, stderr = await asyncio.wait_for(proc.communicate(), 30) except subprocess.TimeoutExpired: proc.kill() raise NurError( From 8d3292af3a233bee9ccbc59aedd8780c4ab962cc Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 20:53:33 -0800 Subject: [PATCH 3/7] Actual true parallel non-blocking updating with asyncio --- bin/nur | 2 +- ci/nur/error.py | 4 ++ ci/nur/eval.py | 20 ++++++---- ci/nur/prefetch.py | 92 +++++++++++++++++++++++++++++----------------- ci/nur/update.py | 68 ++++++++++++++++++++++++---------- 5 files changed, 125 insertions(+), 61 deletions(-) diff --git a/bin/nur b/bin/nur index 998edbeb0..62b8b462d 100755 --- a/bin/nur +++ b/bin/nur @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -p python3 -p nix-prefetch-git -p nix -i python3 +#!nix-shell -p python3 -p python3Packages.aiohttp -p nix-prefetch-git -p nix -i python3 import sys import os diff --git a/ci/nur/error.py b/ci/nur/error.py index fa2665cce..8ac6b0d59 100644 --- a/ci/nur/error.py +++ b/ci/nur/error.py @@ -4,3 +4,7 @@ class NurError(Exception): class EvalError(NurError): pass + + +class RepositoryDeletedError(NurError): + pass diff --git a/ci/nur/eval.py b/ci/nur/eval.py index 69963526f..e33808a85 100644 --- a/ci/nur/eval.py +++ b/ci/nur/eval.py @@ -2,6 +2,7 @@ import logging import os import subprocess import tempfile +import asyncio from argparse import Namespace from pathlib import Path from urllib.parse import urlparse @@ -13,7 +14,7 @@ from .path import EVALREPO_PATH, nixpkgs_path logger = logging.getLogger(__name__) -def eval_repo(repo: Repo, repo_path: Path) -> None: +async def eval_repo(repo: Repo, repo_path: Path) -> None: with tempfile.TemporaryDirectory() as d: eval_path = Path(d).joinpath("default.nix") with open(eval_path, "w") as f: @@ -49,15 +50,20 @@ import {EVALREPO_PATH} {{ ] # fmt: on - logger.info(f"Evaluate repository {repo.name}") env = dict(PATH=os.environ["PATH"], NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM="1") - proc = subprocess.Popen(cmd, env=env, stdout=subprocess.DEVNULL) + proc = await asyncio.create_subprocess_exec( + *cmd, + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) try: - res = proc.wait(15) - except subprocess.TimeoutExpired: + stdout, stderr = await asyncio.wait_for(proc.communicate(), 15) + except TimeoutError: + proc.kill() raise EvalError(f"evaluation for {repo.name} timed out of after 15 seconds") - if res != 0: - raise EvalError(f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}") + if proc.returncode != 0: + raise EvalError(f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}\n\n{stdout.decode()}") async def eval_command(args: Namespace) -> None: diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index fa56735bc..822b2308e 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -3,12 +3,13 @@ import os import re import subprocess import asyncio +import aiohttp from pathlib import Path from typing import Optional, Tuple from urllib.parse import urlparse, ParseResult -from .error import NurError -from .manifest import LockedVersion, Repo, RepoType +from .error import NurError, RepositoryDeletedError +from .manifest import Repo, RepoType Url = ParseResult @@ -29,25 +30,51 @@ async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: return sha256, Path(path) +def parse_pkt_lines(data: bytes): + i = 0 + lines = [] + while i < len(data): + if i + 4 > len(data): + break + length = int(data[i:i+4], 16) + i += 4 + if length == 0: + continue + line = data[i:i+length-4] + i += length - 4 + lines.append(line) + return lines + + class GitPrefetcher: def __init__(self, repo: Repo) -> None: self.repo = repo async def latest_commit(self) -> str: - proc = await asyncio.create_subprocess_exec( - *["git", "ls-remote", self.repo.url.geturl(), self.repo.branch or "HEAD"], - env={**os.environ, "GIT_ASKPASS": "", "GIT_TERMINAL_PROMPT": "0"}, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await proc.communicate() + info_url = f"{self.repo.url.geturl()}/info/refs?service=git-upload-pack" - if proc.returncode != 0: - raise NurError( - f"Failed to prefetch git repository {self.repo.url.geturl()}: {stderr.decode()}" - ) + async with aiohttp.ClientSession() as session: + async with session.get(info_url) as resp: + if resp.status == 401: + raise RepositoryDeletedError(f"Repository deleted!") + elif resp.status != 200: + raise NurError(f"Failed to get refs for {self.repo.url.geturl()}: {(await resp.read()).decode()}") + raw = await resp.read() - return stdout.decode().split(maxsplit=1)[0] + lines = parse_pkt_lines(raw) + + wanted = b"HEAD" if self.repo.branch is None else f"refs/heads/{self.repo.branch}".encode() + + for line in lines: + # Strip capabilities after NUL + if b"\x00" in line: + line = line.split(b"\x00", 1)[0] + + parts = line.strip().split() + if len(parts) == 2 and parts[1] == wanted: + return parts[0].decode() + + raise NurError(f"Ref not found: {wanted.decode()}") async def prefetch(self, ref: str) -> Tuple[str, Path]: cmd = ["nix-prefetch-git"] @@ -63,7 +90,7 @@ class GitPrefetcher: ) try: stdout, stderr = await asyncio.wait_for(proc.communicate(), 30) - except subprocess.TimeoutExpired: + except TimeoutError: proc.kill() raise NurError( f"Timeout expired while prefetching git repository {self. repo.url.geturl()}" @@ -77,8 +104,15 @@ class GitPrefetcher: metadata = json.loads(stdout) lines = stderr.decode("utf-8").split("\n") repo_path = re.search("path is (.+)", lines[-5]) - assert repo_path is not None + if not repo_path: + raise NurError( + f"Failed to prefetch git repository {self.repo.url.geturl()}" + ) path = Path(repo_path.group(1)) + if not path: + raise NurError( + f"Failed to prefetch git repository {self.repo.url.geturl()}" + ) sha256 = metadata["sha256"] return sha256, path @@ -99,21 +133,11 @@ class GitlabPrefetcher(GitPrefetcher): url = f"https://{hostname}/api/v4/projects/{escaped_path}/repository/archive.tar.gz?sha={ref}" return await nix_prefetch_zip(url) - -async def prefetch(repo: Repo) -> Tuple[Repo, LockedVersion, Optional[Path]]: - prefetcher: GitPrefetcher - if repo.type == RepoType.GITHUB: - prefetcher = GithubPrefetcher(repo) - elif repo.type == RepoType.GITLAB: - prefetcher = GitlabPrefetcher(repo) - else: - prefetcher = GitPrefetcher(repo) - - commit = await prefetcher.latest_commit() - locked_version = repo.locked_version - if locked_version is not None: - if locked_version.rev == commit: - return repo, locked_version, None - - sha256, path = await prefetcher.prefetch(commit) - return repo, LockedVersion(repo.url, commit, sha256, repo.submodules), path +def prefetcher_for(repo: Repo) -> GitPrefetcher: + match repo.type: + case RepoType.GITHUB: + return GithubPrefetcher(repo) + case RepoType.GITLAB: + return GithubPrefetcher(repo) + case _: + return GitPrefetcher(repo) diff --git a/ci/nur/update.py b/ci/nur/update.py index f30835b43..40e91a482 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -1,22 +1,27 @@ import logging +import asyncio from argparse import Namespace from concurrent.futures import ThreadPoolExecutor, as_completed from .eval import EvalError, eval_repo -from .manifest import Repo, load_manifest, update_lock_file +from .manifest import Repo, LockedVersion, load_manifest, update_lock_file from .path import LOCK_PATH, MANIFEST_PATH -from .prefetch import prefetch +from .prefetch import prefetcher_for logger = logging.getLogger(__name__) async def update(repo: Repo) -> Repo: - repo, locked_version, repo_path = await prefetch(repo) + prefetcher = prefetcher_for(repo) - if repo_path: - eval_repo(repo, repo_path) + latest_commit = await prefetcher.latest_commit() - repo.locked_version = locked_version + if repo.locked_version is not None and repo.locked_version.rev == latest_commit: + return repo + + sha256, repo_path = await prefetcher.prefetch(latest_commit) + eval_repo(repo, repo_path) + repo.locked_version = LockedVersion(repo.url, latest_commit, sha256, repo.submodules) return repo @@ -25,17 +30,42 @@ async def update_command(args: Namespace) -> None: manifest = load_manifest(MANIFEST_PATH, LOCK_PATH) - for repo in manifest.repos: - try: - await update(repo) - except EvalError as err: - if repo.locked_version is None: - logger.error( - f"repository {repo.name} failed to evaluate: {err}. This repo is not yet in our lock file!!!!" - ) - raise - logger.error(f"repository {repo.name} failed to evaluate: {err}") - except Exception: - logger.exception(f"Failed to update repository {repo.name}") + log_lock = asyncio.Lock() # serialize success/error output - update_lock_file(manifest.repos, LOCK_PATH) + results: List[Tuple[int, Optional[Repo], Optional[BaseException]]] = [] + + async def run_one(i: int, repo: Repo) -> None: + try: + updated = await update(repo) + + results.append((i, updated, None)) + + async with log_lock: + if updated.locked_version is not None: + logger.info(f"Updated repository {repo.name} -> {updated.locked_version.rev}") + else: + logger.info(f"Updated repository {repo.name}") + except BaseException as e: + results.append((i, None, e)) + + async with log_lock: + if isinstance(e, EvalError) and repo.locked_version is None: + logger.error( + f"repository {repo.name} failed to evaluate: {e}. " + "This repo is not yet in our lock file!!!!" + ) + elif isinstance(e, EvalError): + logger.error(f"repository {repo.name} failed to evaluate: {e}") + else: + logger.exception(f"Failed to update repository {repo.name}", exc_info=e) + + tasks = [asyncio.create_task(run_one(i, repo)) for i, repo in enumerate(manifest.repos)] + await asyncio.gather(*tasks) + + updated_repos: List[Repo] = list(manifest.repos) + + for i, updated, err in results: + if err is None and updated is not None: + updated_repos[i] = updated + + update_lock_file(updated_repos, LOCK_PATH) From 146186522cafc24dfef7365b1fade6b351d882f3 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 22:12:09 -0800 Subject: [PATCH 4/7] Some fixes --- ci/nur/eval.py | 2 +- ci/nur/prefetch.py | 4 ++-- ci/nur/update.py | 3 ++- ci/test.sh | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/nur/eval.py b/ci/nur/eval.py index e33808a85..2689872d3 100644 --- a/ci/nur/eval.py +++ b/ci/nur/eval.py @@ -80,5 +80,5 @@ async def eval_command(args: Namespace) -> None: None, None, ) - eval_repo(repo, repo_path) + await eval_repo(repo, repo_path) print("OK") diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index 822b2308e..7d00b0aee 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -5,7 +5,7 @@ import subprocess import asyncio import aiohttp from pathlib import Path -from typing import Optional, Tuple +from typing import Optional, Tuple, List from urllib.parse import urlparse, ParseResult from .error import NurError, RepositoryDeletedError @@ -30,7 +30,7 @@ async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: return sha256, Path(path) -def parse_pkt_lines(data: bytes): +def parse_pkt_lines(data: bytes) -> List[bytes]: i = 0 lines = [] while i < len(data): diff --git a/ci/nur/update.py b/ci/nur/update.py index 40e91a482..0b4d51f3c 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -1,5 +1,6 @@ import logging import asyncio +from typing import List from argparse import Namespace from concurrent.futures import ThreadPoolExecutor, as_completed @@ -20,7 +21,7 @@ async def update(repo: Repo) -> Repo: return repo sha256, repo_path = await prefetcher.prefetch(latest_commit) - eval_repo(repo, repo_path) + await eval_repo(repo, repo_path) repo.locked_version = LockedVersion(repo.url, latest_commit, sha256, repo.submodules) return repo diff --git a/ci/test.sh b/ci/test.sh index d419b0730..f01ff6597 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -p bash -i bash -p mypy -p black -p ruff -p nix +#!nix-shell -p bash -i bash -p mypy -p black -p ruff -p nix -p python3Packages.aiohttp set -eux -o pipefail # Exit with nonzero exit code if anything fails From cf2351662cde3db7e75cfec1c0951db597be497e Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 22:15:12 -0800 Subject: [PATCH 5/7] Fix missing import --- ci/nur/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/nur/update.py b/ci/nur/update.py index 0b4d51f3c..9c7b13aac 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -1,6 +1,6 @@ import logging import asyncio -from typing import List +from typing import List, Tuple from argparse import Namespace from concurrent.futures import ThreadPoolExecutor, as_completed From 542b6747ccd38f0a60f17358fe8c1be3253f24f5 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 22:18:01 -0800 Subject: [PATCH 6/7] Another missing import --- ci/nur/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/nur/update.py b/ci/nur/update.py index 9c7b13aac..647973c4d 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -1,6 +1,6 @@ import logging import asyncio -from typing import List, Tuple +from typing import List, Tuple, Optional from argparse import Namespace from concurrent.futures import ThreadPoolExecutor, as_completed From c33894196a7a4329521490cc910ece12f81a525c Mon Sep 17 00:00:00 2001 From: Gavin John Date: Thu, 29 Jan 2026 22:20:54 -0800 Subject: [PATCH 7/7] Fix formatting --- ci/nur/__init__.py | 2 +- ci/nur/eval.py | 7 ++++--- ci/nur/prefetch.py | 35 ++++++++++++++++++++--------------- ci/nur/update.py | 25 ++++++++++++++++--------- 4 files changed, 41 insertions(+), 28 deletions(-) diff --git a/ci/nur/__init__.py b/ci/nur/__init__.py index 55dddf215..f298b4fff 100644 --- a/ci/nur/__init__.py +++ b/ci/nur/__init__.py @@ -1,7 +1,7 @@ import argparse +import asyncio import logging import sys -import asyncio from typing import List from .combine import combine_command diff --git a/ci/nur/eval.py b/ci/nur/eval.py index 2689872d3..789b08e0c 100644 --- a/ci/nur/eval.py +++ b/ci/nur/eval.py @@ -1,8 +1,7 @@ +import asyncio import logging import os -import subprocess import tempfile -import asyncio from argparse import Namespace from pathlib import Path from urllib.parse import urlparse @@ -63,7 +62,9 @@ import {EVALREPO_PATH} {{ proc.kill() raise EvalError(f"evaluation for {repo.name} timed out of after 15 seconds") if proc.returncode != 0: - raise EvalError(f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}\n\n{stdout.decode()}") + raise EvalError( + f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}\n\n{stdout.decode()}" + ) async def eval_command(args: Namespace) -> None: diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index 7d00b0aee..7e37eb16b 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -1,18 +1,18 @@ -import json -import os -import re -import subprocess import asyncio -import aiohttp +import json +import re from pathlib import Path -from typing import Optional, Tuple, List -from urllib.parse import urlparse, ParseResult +from typing import List, Tuple +from urllib.parse import ParseResult + +import aiohttp from .error import NurError, RepositoryDeletedError from .manifest import Repo, RepoType Url = ParseResult + async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: proc = await asyncio.create_subprocess_exec( *["nix-prefetch-url", "--name", "source", "--unpack", "--print-path", url], @@ -22,9 +22,7 @@ async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: stdout, stderr = await proc.communicate() if proc.returncode != 0: - raise NurError( - f"Failed to prefetch git repository {url}: {stderr.decode()}" - ) + raise NurError(f"Failed to prefetch git repository {url}: {stderr.decode()}") sha256, path = stdout.decode().strip().split("\n") return sha256, Path(path) @@ -36,11 +34,11 @@ def parse_pkt_lines(data: bytes) -> List[bytes]: while i < len(data): if i + 4 > len(data): break - length = int(data[i:i+4], 16) + length = int(data[i : i + 4], 16) i += 4 if length == 0: continue - line = data[i:i+length-4] + line = data[i : i + length - 4] i += length - 4 lines.append(line) return lines @@ -56,14 +54,20 @@ class GitPrefetcher: async with aiohttp.ClientSession() as session: async with session.get(info_url) as resp: if resp.status == 401: - raise RepositoryDeletedError(f"Repository deleted!") + raise RepositoryDeletedError("Repository deleted!") elif resp.status != 200: - raise NurError(f"Failed to get refs for {self.repo.url.geturl()}: {(await resp.read()).decode()}") + raise NurError( + f"Failed to get refs for {self.repo.url.geturl()}: {(await resp.read()).decode()}" + ) raw = await resp.read() lines = parse_pkt_lines(raw) - wanted = b"HEAD" if self.repo.branch is None else f"refs/heads/{self.repo.branch}".encode() + wanted = ( + b"HEAD" + if self.repo.branch is None + else f"refs/heads/{self.repo.branch}".encode() + ) for line in lines: # Strip capabilities after NUL @@ -133,6 +137,7 @@ class GitlabPrefetcher(GitPrefetcher): url = f"https://{hostname}/api/v4/projects/{escaped_path}/repository/archive.tar.gz?sha={ref}" return await nix_prefetch_zip(url) + def prefetcher_for(repo: Repo) -> GitPrefetcher: match repo.type: case RepoType.GITHUB: diff --git a/ci/nur/update.py b/ci/nur/update.py index 647973c4d..6de323e0d 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -1,11 +1,10 @@ -import logging import asyncio -from typing import List, Tuple, Optional +import logging from argparse import Namespace -from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Optional, Tuple from .eval import EvalError, eval_repo -from .manifest import Repo, LockedVersion, load_manifest, update_lock_file +from .manifest import LockedVersion, Repo, load_manifest, update_lock_file from .path import LOCK_PATH, MANIFEST_PATH from .prefetch import prefetcher_for @@ -22,7 +21,9 @@ async def update(repo: Repo) -> Repo: sha256, repo_path = await prefetcher.prefetch(latest_commit) await eval_repo(repo, repo_path) - repo.locked_version = LockedVersion(repo.url, latest_commit, sha256, repo.submodules) + repo.locked_version = LockedVersion( + repo.url, latest_commit, sha256, repo.submodules + ) return repo @@ -31,7 +32,7 @@ async def update_command(args: Namespace) -> None: manifest = load_manifest(MANIFEST_PATH, LOCK_PATH) - log_lock = asyncio.Lock() # serialize success/error output + log_lock = asyncio.Lock() # serialize success/error output results: List[Tuple[int, Optional[Repo], Optional[BaseException]]] = [] @@ -43,7 +44,9 @@ async def update_command(args: Namespace) -> None: async with log_lock: if updated.locked_version is not None: - logger.info(f"Updated repository {repo.name} -> {updated.locked_version.rev}") + logger.info( + f"Updated repository {repo.name} -> {updated.locked_version.rev}" + ) else: logger.info(f"Updated repository {repo.name}") except BaseException as e: @@ -58,9 +61,13 @@ async def update_command(args: Namespace) -> None: elif isinstance(e, EvalError): logger.error(f"repository {repo.name} failed to evaluate: {e}") else: - logger.exception(f"Failed to update repository {repo.name}", exc_info=e) + logger.exception( + f"Failed to update repository {repo.name}", exc_info=e + ) - tasks = [asyncio.create_task(run_one(i, repo)) for i, repo in enumerate(manifest.repos)] + tasks = [ + asyncio.create_task(run_one(i, repo)) for i, repo in enumerate(manifest.repos) + ] await asyncio.gather(*tasks) updated_repos: List[Repo] = list(manifest.repos)