diff --git a/ci/nur/manifest.py b/ci/nur/manifest.py index 346e819b6..215e0b176 100644 --- a/ci/nur/manifest.py +++ b/ci/nur/manifest.py @@ -67,10 +67,7 @@ class Repo: self.file = "default.nix" else: self.file = file_ - if branch is None: - self.branch = "master" - else: - self.branch = branch + self.branch = branch self.locked_version = None if ( @@ -155,7 +152,7 @@ def load_manifest(manifest_path: PathType, lock_path: PathType) -> Manifest: for name, repo in data["repos"].items(): url = urlparse(repo["url"]) submodules = repo.get("submodules", False) - branch_ = repo.get("branch", "master") + branch_ = repo.get("branch") file_ = repo.get("file", "default.nix") type_ = repo.get("type", None) locked_version = locked_versions.get(name) diff --git a/ci/nur/prefetch.py b/ci/nur/prefetch.py index 79d04343b..31313d791 100644 --- a/ci/nur/prefetch.py +++ b/ci/nur/prefetch.py @@ -1,33 +1,15 @@ import json +import os import re import subprocess -import urllib.error -import urllib.request -import xml.etree.ElementTree as ET from pathlib import Path -from typing import List, Optional, Tuple -from urllib.parse import urljoin, urlparse +from typing import Optional, Tuple +from urllib.parse import ParseResult from .error import NurError from .manifest import LockedVersion, Repo, RepoType - -def fetch_commit_from_feed(url: str) -> str: - req = urllib.request.Request(url, headers={"User-Agent": "nur-updater"}) - res = urllib.request.urlopen(req) - try: - xml = res.read() - root = ET.fromstring(xml) - ns = "{http://www.w3.org/2005/Atom}" - xpath = f"./{ns}entry/{ns}link" - commit_link = root.find(xpath) - if commit_link is None: - raise NurError(f"No commits found in repository feed {url}") - return Path(urlparse(commit_link.attrib["href"]).path).parts[-1] - except urllib.error.HTTPError as e: - if e.code == 404: - raise NurError(f"Repository feed {url} not found") - raise +Url = ParseResult def nix_prefetch_zip(url: str) -> Tuple[str, Path]: @@ -38,103 +20,78 @@ def nix_prefetch_zip(url: str) -> Tuple[str, Path]: return sha256, Path(path) -class GithubRepo: - def __init__(self, owner: str, name: str, branch: str) -> None: - self.owner = owner - self.name = name - self.branch = branch - - def url(self, path: str) -> str: - return urljoin(f"https://github.com/{self.owner}/{self.name}/", path) +class GitPrefetcher: + def __init__(self, repo: Repo) -> None: + self.repo = repo def latest_commit(self) -> str: - return fetch_commit_from_feed(self.url(f"commits/{self.branch}.atom")) + data = subprocess.check_output( + ["git", "ls-remote", self.repo.url.geturl(), self.repo.branch or "HEAD"], + env={**os.environ, "GIT_ASKPASS": "", "GIT_TERMINAL_PROMPT": "0"}, + ) + return data.decode().split(maxsplit=1)[0] def prefetch(self, ref: str) -> Tuple[str, Path]: - return nix_prefetch_zip(self.url(f"archive/{ref}.tar.gz")) + cmd = ["nix-prefetch-git"] + if self.repo.submodules: + cmd += ["--fetch-submodules"] + if self.repo.branch: + cmd += ["--rev", f"refs/heads/{self.repo.branch}"] + cmd += [self.repo.url.geturl()] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + stdout, stderr = proc.communicate(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + raise NurError( + f"Timeout expired while prefetching git repository {self. repo.url.geturl()}" + ) + + if proc.returncode != 0: + raise NurError( + f"Failed to prefetch git repository {self.repo.url.geturl()}: {stderr.decode('utf-8')}" + ) + + metadata = json.loads(stdout) + lines = stderr.decode("utf-8").split("\n") + repo_path = re.search("path is (.+)", lines[-5]) + assert repo_path is not None + path = Path(repo_path.group(1)) + sha256 = metadata["sha256"] + return sha256, path -class GitlabRepo: - def __init__(self, domain: str, path: List[str], branch: str) -> None: - self.domain = domain - self.path = path - self.branch = branch - - def latest_commit(self) -> str: - path = "/".join(self.path) - url = f"https://{self.domain}/{path}/commits/{self.branch}?format=atom" - return fetch_commit_from_feed(url) - +class GithubPrefetcher(GitPrefetcher): def prefetch(self, ref: str) -> Tuple[str, Path]: - escaped_path = "%2F".join(self.path) - url = f"https://{self.domain}/api/v4/projects/{escaped_path}/repository/archive.tar.gz?sha={ref}" + return nix_prefetch_zip(f"{self.repo.url.geturl()}/archive/{ref}.tar.gz") + + +class GitlabPrefetcher(GitPrefetcher): + def prefetch(self, ref: str) -> Tuple[str, Path]: + hostname = self.repo.url.hostname + assert ( + hostname is not None + ), f"Expect a hostname for Gitlab repo: {self.repo.name}" + path = Path(self.repo.url.path) + escaped_path = "%2F".join(path.parts[1:]) + url = f"https://{hostname}/api/v4/projects/{escaped_path}/repository/archive.tar.gz?sha={ref}" return nix_prefetch_zip(url) -def prefetch_git(repo: Repo) -> Tuple[LockedVersion, Path]: - cmd = ["nix-prefetch-git"] - if repo.submodules: - cmd += ["--fetch-submodules"] - cmd += ["--rev", f"refs/heads/{repo.branch}"] - cmd += [repo.url.geturl()] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - stdout, stderr = proc.communicate(timeout=30) - except subprocess.TimeoutExpired: - proc.kill() - raise NurError( - f"Timeout expired while prefetching git repository {repo.url.geturl()}" - ) - - if proc.returncode != 0: - raise NurError( - f"Failed to prefetch git repository {repo.url.geturl()}: {stderr.decode('utf-8')}" - ) - - metadata = json.loads(stdout) - lines = stderr.decode("utf-8").split("\n") - repo_path = re.search("path is (.+)", lines[-5]) - assert repo_path is not None - path = Path(repo_path.group(1)) - rev = metadata["rev"] - sha256 = metadata["sha256"] - return LockedVersion(repo.url, rev, sha256, repo.submodules), path - - -def prefetch_github(repo: Repo) -> Tuple[LockedVersion, Optional[Path]]: - github_path = Path(repo.url.path) - gh_repo = GithubRepo(github_path.parts[1], github_path.parts[2], repo.branch) - commit = gh_repo.latest_commit() - locked_version = repo.locked_version - if locked_version is not None: - if locked_version.rev == commit: - return locked_version, None - sha256, path = gh_repo.prefetch(commit) - - return LockedVersion(repo.url, commit, sha256), path - - -def prefetch_gitlab(repo: Repo) -> Tuple[LockedVersion, Optional[Path]]: - gitlab_path = Path(repo.url.path) - hostname = repo.url.hostname - assert hostname is not None, f"Expect a hostname for Gitlab repo: {repo.name}" - gl_repo = GitlabRepo(hostname, list(gitlab_path.parts[1:]), repo.branch) - commit = gl_repo.latest_commit() - locked_version = repo.locked_version - if locked_version is not None: - if locked_version.rev == commit: - return locked_version, None - - sha256, path = gl_repo.prefetch(commit) - return LockedVersion(repo.url, commit, sha256), path - - def prefetch(repo: Repo) -> Tuple[Repo, LockedVersion, Optional[Path]]: + prefetcher: GitPrefetcher if repo.type == RepoType.GITHUB: - locked_version, path = prefetch_github(repo) + prefetcher = GithubPrefetcher(repo) elif repo.type == RepoType.GITLAB: - locked_version, path = prefetch_gitlab(repo) + prefetcher = GitlabPrefetcher(repo) else: - locked_version, path = prefetch_git(repo) + prefetcher = GitPrefetcher(repo) - return repo, locked_version, path + commit = prefetcher.latest_commit() + locked_version = repo.locked_version + if locked_version is not None: + if locked_version.rev == commit: + return repo, locked_version, None + + sha256, path = prefetcher.prefetch(commit) + return repo, LockedVersion(repo.url, commit, sha256, repo.submodules), path diff --git a/repos.json b/repos.json index 30f329b0e..7a2ee6d2e 100644 --- a/repos.json +++ b/repos.json @@ -1,7 +1,6 @@ { "repos": { "0x4A6F": { - "branch": "main", "github-contact": "0x4A6F", "url": "https://github.com/0x4A6F/nur-packages" }, @@ -19,7 +18,6 @@ "url": "https://github.com/afreakk/mynixrepo" }, "alarsyo": { - "branch": "main", "file": "pkgs/default.nix", "github-contact": "alarsyo", "url": "https://github.com/alarsyo/nixos-config" @@ -33,13 +31,11 @@ "url": "https://gitlab.com/AlwinB/nur-packages" }, "ambroisie": { - "branch": "main", "file": "pkgs/default.nix", "github-contact": "ambroisie", "url": "https://github.com/ambroisie/nix-config" }, "amesgen": { - "branch": "main", "github-contact": "amesgen", "url": "https://github.com/amesgen/nur-packages" }, @@ -273,7 +269,6 @@ "url": "https://github.com/htr/nur-packages" }, "hujw77": { - "branch": "main", "github-contact": "hujw77", "url": "https://github.com/hujw77/nur-packages" }, @@ -410,7 +405,6 @@ "url": "https://github.com/lucasew/nixcfg" }, "m15a": { - "branch": "main", "github-contact": "m15a", "submodules": true, "url": "https://github.com/m15a/nur-packages" @@ -449,13 +443,11 @@ "url": "https://github.com/milahu/nur-packages" }, "mipmip": { - "branch": "main", "file": "pkgs/default.nix", "github-contact": "mipmip", "url": "https://github.com/mipmip/nixos" }, "misterio": { - "branch": "main", "github-contact": "misterio77", "type": "sourcehut", "url": "https://git.sr.ht/~misterio/nix-config" @@ -604,7 +596,6 @@ "url": "https://github.com/pniedzwiedzinski/pnpkgs" }, "polykernel": { - "branch": "main", "github-contact": "polykernel", "url": "https://github.com/polykernel/nur-packages" },