Merge pull request #61 from nix-community/channel

implement channel lock files

nur/__init__.py

@@ -17,7 +17,7 @@ def parse_arguments(argv: List[str]) -> argparse.Namespace:
     subparsers = parser.add_subparsers(description="subcommands")
 
     build_channel = subparsers.add_parser("build-channel")
-    build_channel.add_argument('directory')
+    build_channel.add_argument("directory")
     build_channel.set_defaults(func=build_channel_command)
 
     format_manifest = subparsers.add_parser("format-manifest")

nur/channel.py

@@ -1,15 +1,18 @@
+import logging
 import os
 import shutil
 import subprocess
 from argparse import Namespace
 from distutils.dir_util import copy_tree
 from pathlib import Path
-from typing import Dict, Optional, List
+from typing import Dict, List, Optional
 
 from .fileutils import chdir, write_json_file
-from .manifest import Repo, load_manifest
+from .manifest import Repo, load_manifest, update_lock_file
 from .path import LOCK_PATH, MANIFEST_PATH, ROOT
 
+logger = logging.getLogger(__name__)
+
 
 def load_channel_repos(path: Path) -> Dict[str, Repo]:
     channel_manifest = load_manifest(
@@ -22,13 +25,7 @@ def load_channel_repos(path: Path) -> Dict[str, Repo]:
 
 
 def repo_source(name: str) -> str:
-    cmd = [
-        "nix-build",
-        str(ROOT),
-        "--no-out-link",
-        "-A",
-        f"repo-sources.{name}",
-    ]
+    cmd = ["nix-build", str(ROOT), "--no-out-link", "-A", f"repo-sources.{name}"]
     out = subprocess.check_output(cmd)
     return out.strip().decode("utf-8")
 
@@ -46,26 +43,32 @@ def commit_files(files: List[str], message: str) -> None:
         subprocess.check_call(["git", "commit", "-m", message])
 
 
-def commit_repo(repo: Repo, message: str, path: Path) -> None:
+def commit_repo(repo: Repo, message: str, path: Path) -> Repo:
     repo_path = str(path.joinpath(repo.name).resolve())
+
     copy_tree(repo_source(repo.name), repo_path)
 
     with chdir(str(path)):
         commit_files([repo_path], message)
 
+    return repo
 
-def update_channel_repo(channel_repo: Optional[Repo], repo: Repo, path: Path) -> None:
+
+def update_channel_repo(
+    channel_repo: Optional[Repo], repo: Repo, path: Path
+) -> Optional[Repo]:
     if repo.locked_version is None:
-        return
+        return None
 
     new_rev = repo.locked_version.rev
     if channel_repo is None:
         return commit_repo(repo, f"{repo.name}: init at {new_rev}", path)
+
     assert channel_repo.locked_version is not None
     old_rev = channel_repo.locked_version.rev
 
     if channel_repo.locked_version == repo.locked_version:
-        return
+        return repo
 
     if new_rev != new_rev:
         message = f"{repo.name}: {old_rev} -> {new_rev}"
@@ -75,21 +78,41 @@ def update_channel_repo(channel_repo: Optional[Repo], repo: Repo, path: Path) ->
     return commit_repo(repo, message, path)
 
 
+def remove_repo(repo: Repo, path: Path) -> None:
+    repo_path = path.joinpath("repos", repo.name)
+    if repo_path.exists():
+        shutil.rmtree(repo_path)
+    commit_files([str(repo_path)], f"{repo.name}: remove")
+
+
 def update_channel(path: Path) -> None:
     manifest = load_manifest(MANIFEST_PATH, LOCK_PATH)
 
-    old_channel_repos = load_channel_repos(path)
-    channel_repos = old_channel_repos.copy()
+    channel_repos = load_channel_repos(path)
 
     repos_path = path.joinpath("repos")
     os.makedirs(repos_path, exist_ok=True)
 
+    updated_repos = []
+
     for repo in manifest.repos:
         channel_repo = None
         if repo.name in channel_repos:
             channel_repo = channel_repos[repo.name]
             del channel_repos[repo.name]
-        update_channel_repo(channel_repo, repo, repos_path)
+        try:
+            new_repo = update_channel_repo(channel_repo, repo, repos_path)
+        except Exception:
+            logger.exception(f"Failed to updated repository {repo.name}")
+            continue
+
+        if new_repo is not None:
+            updated_repos.append(new_repo)
+
+    for channel_repo in channel_repos.values():
+        remove_repo(channel_repo, path)
+
+    update_lock_file(updated_repos, path.joinpath("repos.json.lock"))
 
 
 def setup_channel() -> None:

nur/fileutils.py

@@ -19,12 +19,12 @@ def to_path(path: PathType) -> Path:
 
 def write_json_file(data: Any, path: PathType) -> None:
     path = to_path(path)
-    f = NamedTemporaryFile(mode='w+', prefix=path.name, dir=str(path.parent))
+    f = NamedTemporaryFile(mode="w+", prefix=path.name, dir=str(path.parent))
     with f as tmp_file:
         json.dump(data, tmp_file, indent=4, sort_keys=True)
         shutil.move(tmp_file.name, path)
         # NamedTemporaryFile tries to delete the file and fails otherwise
-        open(tmp_file.name, 'a').close()
+        open(tmp_file.name, "a").close()
 
 
 @contextmanager

nur/manifest.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Dict, List, Optional, Any
 from urllib.parse import ParseResult, urlparse
 
-from .fileutils import PathType, to_path
+from .fileutils import PathType, to_path, write_json_file
 
 Url = ParseResult
 
@@ -25,9 +25,7 @@ class LockedVersion:
 
     def as_json(self) -> Dict[str, Any]:
         d = dict(
-            url=self.url.geturl(),
-            rev=self.rev,
-            sha256=self.sha256,
+            url=self.url.geturl(), rev=self.rev, sha256=self.sha256
         )  # type: Dict[str, Any]
         if self.submodules:
             d["submodules"] = self.submodules
@@ -113,6 +111,15 @@ def load_locked_versions(path: Path) -> Dict[str, LockedVersion]:
         return {}
 
 
+def update_lock_file(repos: List[Repo], path: Path) -> None:
+    locked_repos = {}
+    for repo in repos:
+        if repo.locked_version:
+            locked_repos[repo.name] = repo.locked_version.as_json()
+
+    write_json_file(dict(repos=locked_repos), path)
+
+
 def load_manifest(manifest_path: PathType, lock_path: PathType) -> Manifest:
     locked_versions = load_locked_versions(to_path(lock_path))
 

nur/update.py

@@ -4,13 +4,11 @@ import subprocess
 import tempfile
 from argparse import Namespace
 from pathlib import Path
-from typing import List
 
 from .error import NurError
-from .manifest import Repo, load_manifest
+from .manifest import Repo, load_manifest, update_lock_file
 from .path import EVALREPO_PATH, LOCK_PATH, MANIFEST_PATH, nixpkgs_path
 from .prefetch import prefetch
-from .fileutils import write_json_file
 
 logger = logging.getLogger(__name__)
 
@@ -68,15 +66,6 @@ def update(repo: Repo) -> Repo:
     return repo
 
 
-def update_lock_file(repos: List[Repo]) -> None:
-    locked_repos = {}
-    for repo in repos:
-        if repo.locked_version:
-            locked_repos[repo.name] = repo.locked_version.as_json()
-
-    write_json_file(dict(repos=locked_repos), LOCK_PATH)
-
-
 def update_command(args: Namespace) -> None:
     manifest = load_manifest(MANIFEST_PATH, LOCK_PATH)
 
@@ -89,4 +78,4 @@ def update_command(args: Namespace) -> None:
                 raise
             logger.exception(f"Failed to updated repository {repo.name}")
 
-    update_lock_file(manifest.repos)
+    update_lock_file(manifest.repos, LOCK_PATH)