add build-channel command

ci/deploy.sh

@@ -2,14 +2,22 @@
 
 set -eu -o pipefail # Exit with nonzero exit code if anything fails
 
-if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] || [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then
-  openssl aes-256-cbc -K $encrypted_025d6e877aa4_key -iv $encrypted_025d6e877aa4_iv -in ci/deploy_key.enc -out deploy_key -d
-  chmod 600 deploy_key
-  eval "$(ssh-agent -s)"
-  ssh-add deploy_key
+add-ssh-key() {
+  key="$1"
+  plain="${key}.plain"
+  openssl aes-256-cbc \
+    -K $encrypted_025d6e877aa4_key -iv $encrypted_025d6e877aa4_iv \
+    -in "$key" -out $plain -d
+  chmod 600 "${key}.plain"
+  ssh-add "${key}.plain"
+  rm "${key}.plain"
+}
 
-  # better safe then sorry
-  rm deploy_key
+if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] || [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then
+  eval "$(ssh-agent -s)"
+
+  add-ssh-key ci/deploy_key.enc
+  add-ssh-key ci/deploy_channel_key.enc
 fi
 
 export encrypted_025d6e877aa4_key= encrypted_025d6e877aa4_iv=
@@ -31,15 +39,24 @@ if [[ "$TRAVIS_EVENT_TYPE" != "cron" ]] && [[ "$TRAVIS_EVENT_TYPE" != "api" ]];
   exit 0
 fi
 
-git config user.name "Travis CI"
-git config user.email "travis@travis.org"
+git config --global user.name "Travis CI"
+git config --global user.email "travis@travis.org"
 
-if [ -z "$(git diff --exit-code)" ]; then
+git clone git@github.com/nix-community/nur-channel
+
+old_channel_rev=$(git rev-parse HEAD)
+./bin/nur build-channel nur-channel
+new_channel_rev=$(git rev-parse HEAD)
+
+if [[ -z "$(git diff --exit-code)" ]]; then
   echo "No changes to the output on this push; exiting."
-  exit 0
+else
+  git add --all repos.json*
+
+  git commit -m "automatic update"
+  git push git@github.com:nix-community/NUR HEAD:master
 fi
 
-git add --all repos.json*
-
-git commit -m "automatic update"
-git push git@github.com:nix-community/NUR HEAD:master
+if [[ $old_channel_rev != $new_channel_rev ]]; then
+  (cd nur-channel && git push origin master)
+fi

lib/repoSource.nix

@@ -17,8 +17,12 @@ let
   revision = lockedRevisions.${name};
   submodules = attr.submodules or false;
   type = attr.type or null;
+
+  localPath = ../repos + "/${name}";
 in
-  if lib.hasPrefix "https://github.com" attr.url && !submodules then
+  if lib.pathExists localPath then
+    localPath
+  else if lib.hasPrefix "https://github.com" attr.url && !submodules then
     fetchzip {
       url = "${attr.url}/archive/${revision.rev}.zip";
       inherit (revision) sha256;

nur/__init__.py

@@ -6,7 +6,7 @@ from .format_manifest import format_manifest_command
 from .index import index_command
 from .update import update_command
 
-# from .build import build_channel_command
+from .channel import build_channel_command
 
 
 def parse_arguments(argv: List[str]) -> argparse.Namespace:
@@ -16,8 +16,9 @@ def parse_arguments(argv: List[str]) -> argparse.Namespace:
 
     subparsers = parser.add_subparsers(description="subcommands")
 
-    # build_channel = subparsers.add_parser("build-channel")
-    # build_channel.set_defaults(func=build_channel_command)
+    build_channel = subparsers.add_parser("build-channel")
+    build_channel.add_argument('directory')
+    build_channel.set_defaults(func=build_channel_command)
 
     format_manifest = subparsers.add_parser("format-manifest")
     format_manifest.set_defaults(func=format_manifest_command)

nur/channel.py

@@ -1,7 +1,120 @@
+import os
+import shutil
+import subprocess
+from argparse import Namespace
+from distutils.dir_util import copy_tree
 from pathlib import Path
+from typing import Dict, Optional, List
 
-from .path import LOCK_PATH, MANIFEST_PATH
+from .fileutils import chdir, write_json_file
+from .manifest import Repo, load_manifest
+from .path import LOCK_PATH, MANIFEST_PATH, ROOT
 
 
-def build_channel_command(_path: str):
-    pass
+def load_channel_repos(path: Path) -> Dict[str, Repo]:
+    channel_manifest = load_manifest(
+        path.joinpath("repos.json"), path.joinpath("repos.json.lock")
+    )
+    repos = {}
+    for repo in channel_manifest.repos:
+        repos[repo.name] = repo
+    return repos
+
+
+def repo_source(name: str) -> str:
+    cmd = [
+        "nix-build",
+        str(ROOT),
+        "--no-out-link",
+        "-A",
+        f"repo-sources.{name}",
+    ]
+    out = subprocess.check_output(cmd)
+    return out.strip().decode("utf-8")
+
+
+def repo_changed() -> bool:
+    diff_cmd = subprocess.Popen(["git", "diff", "--staged", "--exit-code"])
+    return diff_cmd.wait() == 1
+
+
+def commit_files(files: List[str], message: str) -> None:
+    cmd = ["git", "add"]
+    cmd.extend(files)
+    subprocess.check_call(cmd)
+    if repo_changed():
+        subprocess.check_call(["git", "commit", "-m", message])
+
+
+def commit_repo(repo: Repo, message: str, path: Path) -> None:
+    repo_path = str(path.joinpath(repo.name).resolve())
+    copy_tree(repo_source(repo.name), repo_path)
+
+    with chdir(str(path)):
+        commit_files([repo_path], message)
+
+
+def update_channel_repo(channel_repo: Optional[Repo], repo: Repo, path: Path) -> None:
+    if repo.locked_version is None:
+        return
+
+    new_rev = repo.locked_version.rev
+    if channel_repo is None:
+        return commit_repo(repo, f"{repo.name}: init at {new_rev}", path)
+    assert channel_repo.locked_version is not None
+    old_rev = channel_repo.locked_version.rev
+
+    if channel_repo.locked_version == repo.locked_version:
+        return
+
+    if new_rev != new_rev:
+        message = f"{repo.name}: {old_rev} -> {new_rev}"
+    else:
+        message = f"{repo.name}: update"
+
+    return commit_repo(repo, message, path)
+
+
+def update_channel(path: Path) -> None:
+    manifest = load_manifest(MANIFEST_PATH, LOCK_PATH)
+
+    old_channel_repos = load_channel_repos(path)
+    channel_repos = old_channel_repos.copy()
+
+    repos_path = path.joinpath("repos")
+    os.makedirs(repos_path, exist_ok=True)
+
+    for repo in manifest.repos:
+        channel_repo = None
+        if repo.name in channel_repos:
+            channel_repo = channel_repos[repo.name]
+            del channel_repos[repo.name]
+        update_channel_repo(channel_repo, repo, repos_path)
+
+
+def setup_channel() -> None:
+    manifest_path = "repos.json"
+
+    if not Path(".git").exists():
+        cmd = ["git", "init", "."]
+        subprocess.check_call(cmd)
+
+    if not os.path.exists(manifest_path):
+        write_json_file(dict(repos={}), manifest_path)
+
+    manifest_lib = "lib"
+    copy_tree(str(ROOT.joinpath("lib")), manifest_lib)
+    default_nix = "default.nix"
+    shutil.copy(ROOT.joinpath("default.nix"), default_nix)
+
+    vcs_files = [manifest_path, manifest_lib, default_nix]
+
+    commit_files(vcs_files, "update channel code")
+
+
+def build_channel_command(args: Namespace) -> None:
+    channel_path = Path(args.directory)
+
+    with chdir(channel_path):
+        setup_channel()
+    update_channel(channel_path)

nur/fileutils.py

@@ -0,0 +1,37 @@
+import json
+import os
+import shutil
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Any, Union, Generator
+
+
+PathType = Union[str, Path]
+
+
+def to_path(path: PathType) -> Path:
+    if isinstance(path, Path):
+        return path
+    else:
+        return Path(path)
+
+
+def write_json_file(data: Any, path: PathType) -> None:
+    path = to_path(path)
+    f = NamedTemporaryFile(mode='w+', prefix=path.name, dir=str(path.parent))
+    with f as tmp_file:
+        json.dump(data, tmp_file, indent=4, sort_keys=True)
+        shutil.move(tmp_file.name, path)
+        # NamedTemporaryFile tries to delete the file and fails otherwise
+        open(tmp_file.name, 'a').close()
+
+
+@contextmanager
+def chdir(dest: PathType) -> Generator[None, None, None]:
+    previous = os.getcwd()
+    os.chdir(dest)
+    try:
+        yield
+    finally:
+        os.chdir(previous)

nur/manifest.py

@@ -1,9 +1,11 @@
 import json
 from enum import Enum, auto
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Any
 from urllib.parse import ParseResult, urlparse
 
+from .fileutils import PathType, to_path
+
 Url = ParseResult
 
 
@@ -16,12 +18,17 @@ class LockedVersion:
         self.sha256 = sha256
         self.submodules = submodules
 
-    def as_json(self) -> Dict[str, Union[bool, str]]:
+    def __eq__(self, other: Any) -> bool:
+        if type(other) is type(self):
+            return self.__dict__ == other.__dict__
+        return False
+
+    def as_json(self) -> Dict[str, Any]:
         d = dict(
             url=self.url.geturl(),
             rev=self.rev,
             sha256=self.sha256,
-        )
+        )  # type: Dict[str, Any]
         if self.submodules:
             d["submodules"] = self.submodules
         return d
@@ -72,13 +79,19 @@ class Repo:
 
         self.type = RepoType.from_repo(self, type_)
 
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.name}>"
+
 
 class Manifest:
     def __init__(self, repos: List[Repo]) -> None:
         self.repos = repos
 
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {repr(self.repos)}>"
 
-def _load_locked_versions(path: Path) -> Dict[str, LockedVersion]:
+
+def _load_locked_versions(path: PathType) -> Dict[str, LockedVersion]:
     with open(path) as f:
         data = json.load(f)
 
@@ -100,8 +113,8 @@ def load_locked_versions(path: Path) -> Dict[str, LockedVersion]:
         return {}
 
 
-def load_manifest(manifest_path: Union[str, Path], lock_path) -> Manifest:
-    locked_versions = load_locked_versions(lock_path)
+def load_manifest(manifest_path: PathType, lock_path: PathType) -> Manifest:
+    locked_versions = load_locked_versions(to_path(lock_path))
 
     with open(manifest_path) as f:
         data = json.load(f)

nur/prefetch.py

@@ -5,7 +5,7 @@ import urllib.error
 import urllib.request
 import xml.etree.ElementTree as ET
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Optional, Tuple
 from urllib.parse import urljoin, urlparse
 
 from .error import NurError

nur/update.py

@@ -1,7 +1,5 @@
-import json
 import logging
 import os
-import shutil
 import subprocess
 import tempfile
 from argparse import Namespace
@@ -12,6 +10,7 @@ from .error import NurError
 from .manifest import Repo, load_manifest
 from .path import EVALREPO_PATH, LOCK_PATH, MANIFEST_PATH, nixpkgs_path
 from .prefetch import prefetch
+from .fileutils import write_json_file
 
 logger = logging.getLogger(__name__)
 
@@ -69,17 +68,13 @@ def update(repo: Repo) -> Repo:
     return repo
 
 
-def update_lock_file(repos: List[Repo]):
+def update_lock_file(repos: List[Repo]) -> None:
     locked_repos = {}
     for repo in repos:
         if repo.locked_version:
             locked_repos[repo.name] = repo.locked_version.as_json()
 
-    tmp_file = str(LOCK_PATH) + "-new"
-    with open(tmp_file, "w") as lock_file:
-        json.dump(dict(repos=locked_repos), lock_file, indent=4, sort_keys=True)
-
-    shutil.move(tmp_file, LOCK_PATH)
+    write_json_file(locked_repos, LOCK_PATH)
 
 
 def update_command(args: Namespace) -> None: