diff --git a/ci/deploy.sh b/ci/deploy.sh index ee82b8789..118add57f 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -2,14 +2,22 @@ set -eu -o pipefail # Exit with nonzero exit code if anything fails -if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] || [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then - openssl aes-256-cbc -K $encrypted_025d6e877aa4_key -iv $encrypted_025d6e877aa4_iv -in ci/deploy_key.enc -out deploy_key -d - chmod 600 deploy_key - eval "$(ssh-agent -s)" - ssh-add deploy_key +add-ssh-key() { + key="$1" + plain="${key}.plain" + openssl aes-256-cbc \ + -K $encrypted_025d6e877aa4_key -iv $encrypted_025d6e877aa4_iv \ + -in "$key" -out $plain -d + chmod 600 "${key}.plain" + ssh-add "${key}.plain" + rm "${key}.plain" +} - # better safe then sorry - rm deploy_key +if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] || [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then + eval "$(ssh-agent -s)" + + add-ssh-key ci/deploy_key.enc + add-ssh-key ci/deploy_channel_key.enc fi export encrypted_025d6e877aa4_key= encrypted_025d6e877aa4_iv= @@ -31,15 +39,24 @@ if [[ "$TRAVIS_EVENT_TYPE" != "cron" ]] && [[ "$TRAVIS_EVENT_TYPE" != "api" ]]; exit 0 fi -git config user.name "Travis CI" -git config user.email "travis@travis.org" +git config --global user.name "Travis CI" +git config --global user.email "travis@travis.org" -if [ -z "$(git diff --exit-code)" ]; then +git clone git@github.com/nix-community/nur-channel + +old_channel_rev=$(git rev-parse HEAD) +./bin/nur build-channel nur-channel +new_channel_rev=$(git rev-parse HEAD) + +if [[ -z "$(git diff --exit-code)" ]]; then echo "No changes to the output on this push; exiting." - exit 0 +else + git add --all repos.json* + + git commit -m "automatic update" + git push git@github.com:nix-community/NUR HEAD:master fi -git add --all repos.json* - -git commit -m "automatic update" -git push git@github.com:nix-community/NUR HEAD:master +if [[ $old_channel_rev != $new_channel_rev ]]; then + (cd nur-channel && git push origin master) +fi diff --git a/ci/deploy_channel_key.enc b/ci/deploy_channel_key.enc new file mode 100644 index 000000000..46c7e925a Binary files /dev/null and b/ci/deploy_channel_key.enc differ diff --git a/lib/repoSource.nix b/lib/repoSource.nix index 44b7b9868..6bb7b1350 100644 --- a/lib/repoSource.nix +++ b/lib/repoSource.nix @@ -17,8 +17,12 @@ let revision = lockedRevisions.${name}; submodules = attr.submodules or false; type = attr.type or null; + + localPath = ../repos + "/${name}"; in - if lib.hasPrefix "https://github.com" attr.url && !submodules then + if lib.pathExists localPath then + localPath + else if lib.hasPrefix "https://github.com" attr.url && !submodules then fetchzip { url = "${attr.url}/archive/${revision.rev}.zip"; inherit (revision) sha256; diff --git a/nur/__init__.py b/nur/__init__.py index d40072a18..f738471f9 100644 --- a/nur/__init__.py +++ b/nur/__init__.py @@ -6,7 +6,7 @@ from .format_manifest import format_manifest_command from .index import index_command from .update import update_command -# from .build import build_channel_command +from .channel import build_channel_command def parse_arguments(argv: List[str]) -> argparse.Namespace: @@ -16,8 +16,9 @@ def parse_arguments(argv: List[str]) -> argparse.Namespace: subparsers = parser.add_subparsers(description="subcommands") - # build_channel = subparsers.add_parser("build-channel") - # build_channel.set_defaults(func=build_channel_command) + build_channel = subparsers.add_parser("build-channel") + build_channel.add_argument('directory') + build_channel.set_defaults(func=build_channel_command) format_manifest = subparsers.add_parser("format-manifest") format_manifest.set_defaults(func=format_manifest_command) diff --git a/nur/channel.py b/nur/channel.py index 63c45c4d8..4eb26d994 100644 --- a/nur/channel.py +++ b/nur/channel.py @@ -1,7 +1,120 @@ +import os +import shutil +import subprocess +from argparse import Namespace +from distutils.dir_util import copy_tree from pathlib import Path +from typing import Dict, Optional, List -from .path import LOCK_PATH, MANIFEST_PATH +from .fileutils import chdir, write_json_file +from .manifest import Repo, load_manifest +from .path import LOCK_PATH, MANIFEST_PATH, ROOT -def build_channel_command(_path: str): - pass +def load_channel_repos(path: Path) -> Dict[str, Repo]: + channel_manifest = load_manifest( + path.joinpath("repos.json"), path.joinpath("repos.json.lock") + ) + repos = {} + for repo in channel_manifest.repos: + repos[repo.name] = repo + return repos + + +def repo_source(name: str) -> str: + cmd = [ + "nix-build", + str(ROOT), + "--no-out-link", + "-A", + f"repo-sources.{name}", + ] + out = subprocess.check_output(cmd) + return out.strip().decode("utf-8") + + +def repo_changed() -> bool: + diff_cmd = subprocess.Popen(["git", "diff", "--staged", "--exit-code"]) + return diff_cmd.wait() == 1 + + +def commit_files(files: List[str], message: str) -> None: + cmd = ["git", "add"] + cmd.extend(files) + subprocess.check_call(cmd) + if repo_changed(): + subprocess.check_call(["git", "commit", "-m", message]) + + +def commit_repo(repo: Repo, message: str, path: Path) -> None: + repo_path = str(path.joinpath(repo.name).resolve()) + copy_tree(repo_source(repo.name), repo_path) + + with chdir(str(path)): + commit_files([repo_path], message) + + +def update_channel_repo(channel_repo: Optional[Repo], repo: Repo, path: Path) -> None: + if repo.locked_version is None: + return + + new_rev = repo.locked_version.rev + if channel_repo is None: + return commit_repo(repo, f"{repo.name}: init at {new_rev}", path) + assert channel_repo.locked_version is not None + old_rev = channel_repo.locked_version.rev + + if channel_repo.locked_version == repo.locked_version: + return + + if new_rev != new_rev: + message = f"{repo.name}: {old_rev} -> {new_rev}" + else: + message = f"{repo.name}: update" + + return commit_repo(repo, message, path) + + +def update_channel(path: Path) -> None: + manifest = load_manifest(MANIFEST_PATH, LOCK_PATH) + + old_channel_repos = load_channel_repos(path) + channel_repos = old_channel_repos.copy() + + repos_path = path.joinpath("repos") + os.makedirs(repos_path, exist_ok=True) + + for repo in manifest.repos: + channel_repo = None + if repo.name in channel_repos: + channel_repo = channel_repos[repo.name] + del channel_repos[repo.name] + update_channel_repo(channel_repo, repo, repos_path) + + +def setup_channel() -> None: + manifest_path = "repos.json" + + if not Path(".git").exists(): + cmd = ["git", "init", "."] + subprocess.check_call(cmd) + + if not os.path.exists(manifest_path): + write_json_file(dict(repos={}), manifest_path) + + manifest_lib = "lib" + copy_tree(str(ROOT.joinpath("lib")), manifest_lib) + default_nix = "default.nix" + shutil.copy(ROOT.joinpath("default.nix"), default_nix) + + vcs_files = [manifest_path, manifest_lib, default_nix] + + commit_files(vcs_files, "update channel code") + + +def build_channel_command(args: Namespace) -> None: + channel_path = Path(args.directory) + + with chdir(channel_path): + setup_channel() + update_channel(channel_path) diff --git a/nur/fileutils.py b/nur/fileutils.py new file mode 100644 index 000000000..641ce960d --- /dev/null +++ b/nur/fileutils.py @@ -0,0 +1,37 @@ +import json +import os +import shutil +from contextlib import contextmanager +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any, Union, Generator + + +PathType = Union[str, Path] + + +def to_path(path: PathType) -> Path: + if isinstance(path, Path): + return path + else: + return Path(path) + + +def write_json_file(data: Any, path: PathType) -> None: + path = to_path(path) + f = NamedTemporaryFile(mode='w+', prefix=path.name, dir=str(path.parent)) + with f as tmp_file: + json.dump(data, tmp_file, indent=4, sort_keys=True) + shutil.move(tmp_file.name, path) + # NamedTemporaryFile tries to delete the file and fails otherwise + open(tmp_file.name, 'a').close() + + +@contextmanager +def chdir(dest: PathType) -> Generator[None, None, None]: + previous = os.getcwd() + os.chdir(dest) + try: + yield + finally: + os.chdir(previous) diff --git a/nur/manifest.py b/nur/manifest.py index b23ea4bf9..3eaad8e5e 100644 --- a/nur/manifest.py +++ b/nur/manifest.py @@ -1,9 +1,11 @@ import json from enum import Enum, auto from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Union, Any from urllib.parse import ParseResult, urlparse +from .fileutils import PathType, to_path + Url = ParseResult @@ -16,12 +18,17 @@ class LockedVersion: self.sha256 = sha256 self.submodules = submodules - def as_json(self) -> Dict[str, Union[bool, str]]: + def __eq__(self, other: Any) -> bool: + if type(other) is type(self): + return self.__dict__ == other.__dict__ + return False + + def as_json(self) -> Dict[str, Any]: d = dict( url=self.url.geturl(), rev=self.rev, sha256=self.sha256, - ) + ) # type: Dict[str, Any] if self.submodules: d["submodules"] = self.submodules return d @@ -72,13 +79,19 @@ class Repo: self.type = RepoType.from_repo(self, type_) + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + class Manifest: def __init__(self, repos: List[Repo]) -> None: self.repos = repos + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {repr(self.repos)}>" -def _load_locked_versions(path: Path) -> Dict[str, LockedVersion]: + +def _load_locked_versions(path: PathType) -> Dict[str, LockedVersion]: with open(path) as f: data = json.load(f) @@ -100,8 +113,8 @@ def load_locked_versions(path: Path) -> Dict[str, LockedVersion]: return {} -def load_manifest(manifest_path: Union[str, Path], lock_path) -> Manifest: - locked_versions = load_locked_versions(lock_path) +def load_manifest(manifest_path: PathType, lock_path: PathType) -> Manifest: + locked_versions = load_locked_versions(to_path(lock_path)) with open(manifest_path) as f: data = json.load(f) diff --git a/nur/prefetch.py b/nur/prefetch.py index 1fb917a43..255b1b6d2 100644 --- a/nur/prefetch.py +++ b/nur/prefetch.py @@ -5,7 +5,7 @@ import urllib.error import urllib.request import xml.etree.ElementTree as ET from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from typing import Optional, Tuple from urllib.parse import urljoin, urlparse from .error import NurError diff --git a/nur/update.py b/nur/update.py index b7eb3dbf6..81c1d12cd 100644 --- a/nur/update.py +++ b/nur/update.py @@ -1,7 +1,5 @@ -import json import logging import os -import shutil import subprocess import tempfile from argparse import Namespace @@ -12,6 +10,7 @@ from .error import NurError from .manifest import Repo, load_manifest from .path import EVALREPO_PATH, LOCK_PATH, MANIFEST_PATH, nixpkgs_path from .prefetch import prefetch +from .fileutils import write_json_file logger = logging.getLogger(__name__) @@ -69,17 +68,13 @@ def update(repo: Repo) -> Repo: return repo -def update_lock_file(repos: List[Repo]): +def update_lock_file(repos: List[Repo]) -> None: locked_repos = {} for repo in repos: if repo.locked_version: locked_repos[repo.name] = repo.locked_version.as_json() - tmp_file = str(LOCK_PATH) + "-new" - with open(tmp_file, "w") as lock_file: - json.dump(dict(repos=locked_repos), lock_file, indent=4, sort_keys=True) - - shutil.move(tmp_file, LOCK_PATH) + write_json_file(locked_repos, LOCK_PATH) def update_command(args: Namespace) -> None: