diff --git a/ci/nur/__init__.py b/ci/nur/__init__.py index 55dddf215..f298b4fff 100644 --- a/ci/nur/__init__.py +++ b/ci/nur/__init__.py @@ -1,7 +1,7 @@ import argparse +import asyncio import logging import sys -import asyncio from typing import List from .combine import combine_command diff --git a/ci/nur/eval.py b/ci/nur/eval.py index 2689872d3..789b08e0c 100644 --- a/ci/nur/eval.py +++ b/ci/nur/eval.py @@ -1,8 +1,7 @@ +import asyncio import logging import os -import subprocess import tempfile -import asyncio from argparse import Namespace from pathlib import Path from urllib.parse import urlparse @@ -63,7 +62,9 @@ import {EVALREPO_PATH} {{ proc.kill() raise EvalError(f"evaluation for {repo.name} timed out of after 15 seconds") if proc.returncode != 0: - raise EvalError(f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}\n\n{stdout.decode()}") + raise EvalError( + f"{repo.name} does not evaluate:\n$ {' '.join(cmd)}\n\n{stdout.decode()}" + ) async def eval_command(args: Namespace) -> None: diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index 7d00b0aee..7e37eb16b 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -1,18 +1,18 @@ -import json -import os -import re -import subprocess import asyncio -import aiohttp +import json +import re from pathlib import Path -from typing import Optional, Tuple, List -from urllib.parse import urlparse, ParseResult +from typing import List, Tuple +from urllib.parse import ParseResult + +import aiohttp from .error import NurError, RepositoryDeletedError from .manifest import Repo, RepoType Url = ParseResult + async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: proc = await asyncio.create_subprocess_exec( *["nix-prefetch-url", "--name", "source", "--unpack", "--print-path", url], @@ -22,9 +22,7 @@ async def nix_prefetch_zip(url: str) -> Tuple[str, Path]: stdout, stderr = await proc.communicate() if proc.returncode != 0: - raise NurError( - f"Failed to prefetch git repository {url}: {stderr.decode()}" - ) + raise NurError(f"Failed to prefetch git repository {url}: {stderr.decode()}") sha256, path = stdout.decode().strip().split("\n") return sha256, Path(path) @@ -36,11 +34,11 @@ def parse_pkt_lines(data: bytes) -> List[bytes]: while i < len(data): if i + 4 > len(data): break - length = int(data[i:i+4], 16) + length = int(data[i : i + 4], 16) i += 4 if length == 0: continue - line = data[i:i+length-4] + line = data[i : i + length - 4] i += length - 4 lines.append(line) return lines @@ -56,14 +54,20 @@ class GitPrefetcher: async with aiohttp.ClientSession() as session: async with session.get(info_url) as resp: if resp.status == 401: - raise RepositoryDeletedError(f"Repository deleted!") + raise RepositoryDeletedError("Repository deleted!") elif resp.status != 200: - raise NurError(f"Failed to get refs for {self.repo.url.geturl()}: {(await resp.read()).decode()}") + raise NurError( + f"Failed to get refs for {self.repo.url.geturl()}: {(await resp.read()).decode()}" + ) raw = await resp.read() lines = parse_pkt_lines(raw) - wanted = b"HEAD" if self.repo.branch is None else f"refs/heads/{self.repo.branch}".encode() + wanted = ( + b"HEAD" + if self.repo.branch is None + else f"refs/heads/{self.repo.branch}".encode() + ) for line in lines: # Strip capabilities after NUL @@ -133,6 +137,7 @@ class GitlabPrefetcher(GitPrefetcher): url = f"https://{hostname}/api/v4/projects/{escaped_path}/repository/archive.tar.gz?sha={ref}" return await nix_prefetch_zip(url) + def prefetcher_for(repo: Repo) -> GitPrefetcher: match repo.type: case RepoType.GITHUB: diff --git a/ci/nur/update.py b/ci/nur/update.py index 647973c4d..6de323e0d 100644 --- a/ci/nur/update.py +++ b/ci/nur/update.py @@ -1,11 +1,10 @@ -import logging import asyncio -from typing import List, Tuple, Optional +import logging from argparse import Namespace -from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Optional, Tuple from .eval import EvalError, eval_repo -from .manifest import Repo, LockedVersion, load_manifest, update_lock_file +from .manifest import LockedVersion, Repo, load_manifest, update_lock_file from .path import LOCK_PATH, MANIFEST_PATH from .prefetch import prefetcher_for @@ -22,7 +21,9 @@ async def update(repo: Repo) -> Repo: sha256, repo_path = await prefetcher.prefetch(latest_commit) await eval_repo(repo, repo_path) - repo.locked_version = LockedVersion(repo.url, latest_commit, sha256, repo.submodules) + repo.locked_version = LockedVersion( + repo.url, latest_commit, sha256, repo.submodules + ) return repo @@ -31,7 +32,7 @@ async def update_command(args: Namespace) -> None: manifest = load_manifest(MANIFEST_PATH, LOCK_PATH) - log_lock = asyncio.Lock() # serialize success/error output + log_lock = asyncio.Lock() # serialize success/error output results: List[Tuple[int, Optional[Repo], Optional[BaseException]]] = [] @@ -43,7 +44,9 @@ async def update_command(args: Namespace) -> None: async with log_lock: if updated.locked_version is not None: - logger.info(f"Updated repository {repo.name} -> {updated.locked_version.rev}") + logger.info( + f"Updated repository {repo.name} -> {updated.locked_version.rev}" + ) else: logger.info(f"Updated repository {repo.name}") except BaseException as e: @@ -58,9 +61,13 @@ async def update_command(args: Namespace) -> None: elif isinstance(e, EvalError): logger.error(f"repository {repo.name} failed to evaluate: {e}") else: - logger.exception(f"Failed to update repository {repo.name}", exc_info=e) + logger.exception( + f"Failed to update repository {repo.name}", exc_info=e + ) - tasks = [asyncio.create_task(run_one(i, repo)) for i, repo in enumerate(manifest.repos)] + tasks = [ + asyncio.create_task(run_one(i, repo)) for i, repo in enumerate(manifest.repos) + ] await asyncio.gather(*tasks) updated_repos: List[Repo] = list(manifest.repos)