Merge pull request #390 from nix-community/nur

Modernize CI

.github/CODEOWNERS

@@ -1,19 +0,0 @@
-# CODEOWNERS file
-#
-# For documentation on this file,
-# see https://help.github.com/articles/about-codeowners/
-# Mentioned users will get code review requests.
-
-# This file
-/.github/CODEOWNERS @Mic92
-
-# python
-/bin @Mic92
-/nur @Mic92
-/setup.py @Mic92
-
-# CI
-/ci @Mic92 @zimbatm
-
-# nix
-/lib @Infinisil

.github/workflows/pr.yml

@@ -9,4 +9,6 @@ jobs:
     - uses: cachix/install-nix-action@v14
       with:
         nix_path: nixpkgs=channel:nixos-unstable
+        extra_nix_config: |
+          experimental-features = nix-command flakes
     - run: ./ci/test.sh

.github/workflows/update.yml

@@ -17,6 +17,8 @@ jobs:
     - uses: cachix/install-nix-action@v14
       with:
         nix_path: nixpkgs=channel:nixos-unstable
+        extra_nix_config: |
+          experimental-features = nix-command flakes
     - name: update nur / nur-combined
       run: ./ci/update-nur.sh
       env:
@@ -29,6 +31,8 @@ jobs:
     - uses: cachix/install-nix-action@v14
       with:
         nix_path: nixpkgs=channel:nixos-unstable
+        extra_nix_config: |
+          experimental-features = nix-command flakes
     - name: update nur-search/data/packages.json
       run: ./ci/update-nur-search.sh
       env:

bin/nur

@@ -1,10 +1,10 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -p python3 -p python3.pkgs.irc -p nix-prefetch-git -p nix -i python3
+#!nix-shell -p python3 -p nix-prefetch-git -p nix -i python3
 import sys
 import os
 
 sys.path.insert(0, os.path.join(
-    os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+    os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "ci"))
 
 from nur import main
 

ci/flake.lock

@@ -0,0 +1,43 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1631561581,
+        "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1633329294,
+        "narHash": "sha256-0LpQLS4KMgxslMgmDHmxG/5twFlXDBW9z4Or1iOrCvU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "ee084c02040e864eeeb4cf4f8538d92f7c675671",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

ci/flake.nix

@@ -0,0 +1,12 @@
+{
+  description = "Internal flake for ci tasks of NUR";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system: {
+      packages.nur = nixpkgs.legacyPackages.${system}.python3.pkgs.callPackage ./nur.nix {};
+      defaultPackage = self.packages.${system}.nur;
+    });
+}

ci/nur.nix

@@ -0,0 +1,13 @@
+{ buildPythonApplication, lib, nix-prefetch-git, git, nixUnstable, glibcLocales }:
+
+buildPythonApplication {
+  name = "nur";
+  src = ./.;
+
+  doCheck = false;
+
+  makeWrapperArgs = [
+    "--prefix" "PATH" ":" "${lib.makeBinPath [ nix-prefetch-git git nixUnstable ]}"
+    "--set" "LOCALE_ARCHIVE" "${glibcLocales}/lib/locale/locale-archive"
+  ];
+}

ci/nur/__init__.py

@@ -28,11 +28,6 @@ def parse_arguments(argv: List[str]) -> argparse.Namespace:
     subparsers = parser.add_subparsers(description="subcommands")
 
     combine = subparsers.add_parser("combine")
-    combine.add_argument(
-        "--irc-notify",
-        type=str,
-        help="Example nur-bot@chat.freenode.net:6697/nixos-nur",
-    )
     combine.add_argument("directory")
     combine.set_defaults(func=combine_command)
 

ci/nur/combine.py

@@ -6,12 +6,11 @@ from argparse import Namespace
 from distutils.dir_util import copy_tree
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional
 
 from .fileutils import chdir, write_json_file
 from .manifest import Repo, load_manifest, update_lock_file
 from .path import LOCK_PATH, MANIFEST_PATH, ROOT
-from .irc_notify import send
 
 logger = logging.getLogger(__name__)
 
@@ -74,22 +73,21 @@ def repo_link(path: Path) -> str:
 
 def update_combined_repo(
     combined_repo: Optional[Repo], repo: Repo, path: Path
-) -> Tuple[Optional[Repo], Optional[str]]:
+) -> Optional[Repo]:
     if repo.locked_version is None:
-        return None, None
+        return None
 
     new_rev = repo.locked_version.rev
     if combined_repo is None:
         message = f"{repo.name}: init at {new_rev[:10]} ({repo_link(path)})"
         repo = commit_repo(repo, message, path)
-        message += f" ({repo_link(path)})"
-        return repo, message
+        return repo
 
     assert combined_repo.locked_version is not None
     old_rev = combined_repo.locked_version.rev
 
     if combined_repo.locked_version == repo.locked_version:
-        return repo, None
+        return repo
 
     if new_rev != old_rev:
         message = f"{repo.name}: {old_rev[:10]} -> {new_rev[:10]}"
@@ -97,8 +95,7 @@ def update_combined_repo(
         message = f"{repo.name}: update"
 
     repo = commit_repo(repo, message, path)
-    message += f" ({repo_link(path)})"
-    return repo, message
+    return repo
 
 
 def remove_repo(repo: Repo, path: Path) -> None:
@@ -117,7 +114,7 @@ def update_manifest(repos: List[Repo], path: Path) -> None:
     write_json_file(dict(repos=d), path)
 
 
-def update_combined(path: Path) -> List[str]:
+def update_combined(path: Path) -> None:
     manifest = load_manifest(MANIFEST_PATH, LOCK_PATH)
 
     combined_repos = load_combined_repos(path)
@@ -126,7 +123,6 @@ def update_combined(path: Path) -> List[str]:
     os.makedirs(repos_path, exist_ok=True)
 
     updated_repos = []
-    notifications = []
 
     for repo in manifest.repos:
         combined_repo = None
@@ -134,17 +130,13 @@ def update_combined(path: Path) -> List[str]:
             combined_repo = combined_repos[repo.name]
             del combined_repos[repo.name]
         try:
-            new_repo, notification = update_combined_repo(
-                combined_repo, repo, repos_path
-            )
+            new_repo = update_combined_repo(combined_repo, repo, repos_path)
         except Exception:
             logger.exception(f"Failed to updated repository {repo.name}")
             continue
 
         if new_repo is not None:
             updated_repos.append(new_repo)
-        if notification is not None:
-            notifications.append(notification)
 
     for combined_repo in combined_repos.values():
         remove_repo(combined_repo, path)
@@ -156,8 +148,6 @@ def update_combined(path: Path) -> List[str]:
     with chdir(path):
         commit_files(["repos.json", "repos.json.lock"], "update repos.json + lock")
 
-    return notifications
-
 
 def setup_combined() -> None:
     manifest_path = "repos.json"
@@ -184,10 +174,4 @@ def combine_command(args: Namespace) -> None:
 
     with chdir(combined_path):
         setup_combined()
-    notifications = update_combined(combined_path)
-
-    if args.irc_notify:
-        try:
-            send(args.irc_notify, notifications)
-        except Exception as e:
-            print(f"failed to send irc notifications: {e}")
+    update_combined(combined_path)

ci/setup.py

@@ -15,8 +15,7 @@ setup(
     packages=find_packages(),
     entry_points={"console_scripts": ["nur = nur:main"]},
     extras_require={
-        "dev": ["mypy", "flake8>=3.5,<3.6", "black"],
-        "irc-notifications": ["irc"],
+        "dev": ["mypy", "flake8", "black"],
     },
     classifiers=[
         "Development Status :: 5 - Production/Stable",

ci/test.sh

@@ -1,16 +1,11 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -p bash -i bash -p python3Packages.mypy -p python3Packages.black -p python3Packages.flake8 -p nix
+#!nix-shell -p bash -i bash -p python3Packages.mypy -p python3Packages.black -p python3Packages.flake8 -p nixUnstable
 
 set -eux -o pipefail # Exit with nonzero exit code if anything fails
 
-nix run '(import ./release.nix {})' -c nur format-manifest
-if [ -n "$(git diff --exit-code repos.json)" ]; then
-    echo "repos.json was not formatted before committing repos.json:" >&2
-    git diff --exit-code repos.json
-    echo "Please run ./bin/nur/format-manifest.py and updates repos.json accordingly!" >&2
-    exit 1
-fi
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
+cd "${DIR}"
 # Type checker
 mypy nur
 # Format checker
@@ -18,5 +13,14 @@ black --check .
 # Linter
 flake8 .
 
-nix run '(import ./release.nix {})' -c nur update
+cd "${DIR}/.."
+nix run "${DIR}#" -- format-manifest
+if [ -n "$(git diff --exit-code repos.json)" ]; then
+    echo "repos.json was not formatted before committing repos.json:" >&2
+    git diff --exit-code repos.json
+    echo "Please run ./bin/nur/format-manifest.py and updates repos.json accordingly!" >&2
+    exit 1
+fi
+
+nix run "${DIR}#" -- update
 nix-build

ci/update-nur-search.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -p git -p nix -p bash -i bash
+#!nix-shell -p git -p nixUnstable -p bash -i bash
 
 set -eu -o pipefail # Exit with nonzero exit code if anything fails
 
@@ -12,13 +12,15 @@ set -x
 # build package.json for nur-search
 # ---------------------------------
 
-nix-build --quiet release.nix
+nix build "${DIR}#"
 
-git clone --single-branch "https://$API_TOKEN_GITHUB@github.com/nix-community/nur-combined.git"
+cd "${DIR}/.."
 
-git clone --recurse-submodules "https://$API_TOKEN_GITHUB@github.com/nix-community/nur-search.git"
+git clone --single-branch "https://${API_TOKEN_GITHUB:-git}@github.com/nix-community/nur-combined.git" || git -C nur-combined pull
 
-nix run '(import ./release.nix {})' -c nur index nur-combined > nur-search/data/packages.json
+git clone --recurse-submodules "https://${API_TOKEN_GITHUB:-git}@github.com/nix-community/nur-search.git" || git -C nur-search pull
+
+nix run "${DIR}#" -- index nur-combined > nur-search/data/packages.json
 
 # rebuild and publish nur-search repository
 # -----------------------------------------

ci/update-nur.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -p git -p nix -p bash -i bash
+#!nix-shell -p git -p nixUnstable -p bash -i bash
 
 set -eu -o pipefail # Exit with nonzero exit code if anything fails
 
@@ -9,17 +9,14 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 source ${DIR}/lib/setup-git.sh
 set -x
 
-nix run '(import ./release.nix {})' -c nur update
-nix-build
+nix run ${DIR}# -- update
+
+cd ${DIR}/..
 
 git clone \
   --single-branch \
   "https://$API_TOKEN_GITHUB@github.com/nix-community/nur-combined.git"
 
-nix run '(import ./release.nix {})' -c nur combine \
-  --irc-notify nur-bot@chat.freenode.net:6697/nixos-nur \
-  nur-combined
-
 if [[ -z "$(git diff --exit-code)" ]]; then
   echo "No changes to the output on this push; exiting."
 else

nur/irc_notify.py

@@ -1,75 +0,0 @@
-import socket
-import ssl
-from typing import List, Optional
-from urllib.parse import urlparse
-
-
-def notify_irc(
-    server: str,
-    nick: str,
-    password: Optional[str],
-    channel: str,
-    tls: bool = True,
-    port: int = 6697,
-    messages: List[str] = [],
-) -> None:
-    if not messages:
-        return
-
-    sock = socket.socket()
-    if tls:
-        sock = ssl.wrap_socket(
-            sock, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_TLSv1_2
-        )
-
-    def _send(command: str) -> int:
-        return sock.send((f"{command}\r\n").encode())
-
-    sock.connect((server, port))
-    if password:
-        _send(f"PASS {password}")
-    _send(f"NICK {nick}")
-    _send(f"USER {nick} {server} bla :{nick}")
-    _send(f"JOIN :{channel}")
-
-    for m in messages:
-        _send(f"PRIVMSG {channel} :{m}")
-
-    _send("INFO")
-
-    while True:
-        data = sock.recv(4096)
-        if not data:
-            raise RuntimeError("Received empty data")
-
-        # Assume INFO reply means we are done
-        if b"End of /INFO list" in data:
-            break
-
-        if data.startswith(b"PING"):
-            sock.send(data.replace(b"PING", b"PONG"))
-
-    sock.send(b"QUIT")
-    sock.close()
-
-
-def send(url: str, notifications: List[str]) -> None:
-    parsed = urlparse(f"http://{url}")
-    username = parsed.username or "nur-bot"
-    server = parsed.hostname or "chat.freenode.de"
-    if parsed.path != "/" or parsed.path == "":
-        channel = f"#{parsed.path[1:]}"
-    else:
-        channel = "#nixos-nur"
-    port = parsed.port or 6697
-    password = parsed.password
-    if len(notifications) == 0:
-        return
-    notify_irc(
-        server=server,
-        nick=username,
-        password=password,
-        channel=channel,
-        port=port,
-        messages=notifications,
-    )

release.nix

@@ -1,15 +0,0 @@
-{ nixpkgs ? import <nixpkgs> }:
-
-with import <nixpkgs> {};
-
-python3Packages.buildPythonApplication {
-  name = "nur";
-  src = ./.;
-
-  doCheck = true;
-
-  makeWrapperArgs = [
-    "--prefix" "PATH" ":" "${pkgs.lib.makeBinPath [ nix-prefetch-git git nix ]}"
-    "--set" "LOCALE_ARCHIVE" "${glibcLocales}/lib/locale/locale-archive"
-  ];
-}