diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 3db264516..000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,19 +0,0 @@ -# CODEOWNERS file -# -# For documentation on this file, -# see https://help.github.com/articles/about-codeowners/ -# Mentioned users will get code review requests. - -# This file -/.github/CODEOWNERS @Mic92 - -# python -/bin @Mic92 -/nur @Mic92 -/setup.py @Mic92 - -# CI -/ci @Mic92 @zimbatm - -# nix -/lib @Infinisil diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c15d5b508..ca0b37af6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -9,4 +9,6 @@ jobs: - uses: cachix/install-nix-action@v14 with: nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + experimental-features = nix-command flakes - run: ./ci/test.sh diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index da843ccf6..f20218f96 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -17,6 +17,8 @@ jobs: - uses: cachix/install-nix-action@v14 with: nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + experimental-features = nix-command flakes - name: update nur / nur-combined run: ./ci/update-nur.sh env: @@ -29,6 +31,8 @@ jobs: - uses: cachix/install-nix-action@v14 with: nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + experimental-features = nix-command flakes - name: update nur-search/data/packages.json run: ./ci/update-nur-search.sh env: diff --git a/bin/nur b/bin/nur index 3bb1db39d..998edbeb0 100755 --- a/bin/nur +++ b/bin/nur @@ -1,10 +1,10 @@ #!/usr/bin/env nix-shell -#!nix-shell -p python3 -p python3.pkgs.irc -p nix-prefetch-git -p nix -i python3 +#!nix-shell -p python3 -p nix-prefetch-git -p nix -i python3 import sys import os sys.path.insert(0, os.path.join( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "ci")) from nur import main diff --git a/ci/flake.lock b/ci/flake.lock new file mode 100644 index 000000000..70ea91baf --- /dev/null +++ b/ci/flake.lock @@ -0,0 +1,43 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1631561581, + "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1633329294, + "narHash": "sha256-0LpQLS4KMgxslMgmDHmxG/5twFlXDBW9z4Or1iOrCvU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ee084c02040e864eeeb4cf4f8538d92f7c675671", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/ci/flake.nix b/ci/flake.nix new file mode 100644 index 000000000..bf7623922 --- /dev/null +++ b/ci/flake.nix @@ -0,0 +1,12 @@ +{ + description = "Internal flake for ci tasks of NUR"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: { + packages.nur = nixpkgs.legacyPackages.${system}.python3.pkgs.callPackage ./nur.nix {}; + defaultPackage = self.packages.${system}.nur; + }); +} diff --git a/ci/keys.tar.gz.enc b/ci/keys.tar.gz.enc deleted file mode 100644 index 43664bbd8..000000000 Binary files a/ci/keys.tar.gz.enc and /dev/null differ diff --git a/ci/nur.nix b/ci/nur.nix new file mode 100644 index 000000000..c53260ae0 --- /dev/null +++ b/ci/nur.nix @@ -0,0 +1,13 @@ +{ buildPythonApplication, lib, nix-prefetch-git, git, nixUnstable, glibcLocales }: + +buildPythonApplication { + name = "nur"; + src = ./.; + + doCheck = false; + + makeWrapperArgs = [ + "--prefix" "PATH" ":" "${lib.makeBinPath [ nix-prefetch-git git nixUnstable ]}" + "--set" "LOCALE_ARCHIVE" "${glibcLocales}/lib/locale/locale-archive" + ]; +} diff --git a/nur/__init__.py b/ci/nur/__init__.py similarity index 91% rename from nur/__init__.py rename to ci/nur/__init__.py index fab7ca290..ff03df55d 100644 --- a/nur/__init__.py +++ b/ci/nur/__init__.py @@ -28,11 +28,6 @@ def parse_arguments(argv: List[str]) -> argparse.Namespace: subparsers = parser.add_subparsers(description="subcommands") combine = subparsers.add_parser("combine") - combine.add_argument( - "--irc-notify", - type=str, - help="Example nur-bot@chat.freenode.net:6697/nixos-nur", - ) combine.add_argument("directory") combine.set_defaults(func=combine_command) diff --git a/nur/combine.py b/ci/nur/combine.py similarity index 85% rename from nur/combine.py rename to ci/nur/combine.py index 365429ea9..4003195d6 100644 --- a/nur/combine.py +++ b/ci/nur/combine.py @@ -6,12 +6,11 @@ from argparse import Namespace from distutils.dir_util import copy_tree from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional from .fileutils import chdir, write_json_file from .manifest import Repo, load_manifest, update_lock_file from .path import LOCK_PATH, MANIFEST_PATH, ROOT -from .irc_notify import send logger = logging.getLogger(__name__) @@ -74,22 +73,21 @@ def repo_link(path: Path) -> str: def update_combined_repo( combined_repo: Optional[Repo], repo: Repo, path: Path -) -> Tuple[Optional[Repo], Optional[str]]: +) -> Optional[Repo]: if repo.locked_version is None: - return None, None + return None new_rev = repo.locked_version.rev if combined_repo is None: message = f"{repo.name}: init at {new_rev[:10]} ({repo_link(path)})" repo = commit_repo(repo, message, path) - message += f" ({repo_link(path)})" - return repo, message + return repo assert combined_repo.locked_version is not None old_rev = combined_repo.locked_version.rev if combined_repo.locked_version == repo.locked_version: - return repo, None + return repo if new_rev != old_rev: message = f"{repo.name}: {old_rev[:10]} -> {new_rev[:10]}" @@ -97,8 +95,7 @@ def update_combined_repo( message = f"{repo.name}: update" repo = commit_repo(repo, message, path) - message += f" ({repo_link(path)})" - return repo, message + return repo def remove_repo(repo: Repo, path: Path) -> None: @@ -117,7 +114,7 @@ def update_manifest(repos: List[Repo], path: Path) -> None: write_json_file(dict(repos=d), path) -def update_combined(path: Path) -> List[str]: +def update_combined(path: Path) -> None: manifest = load_manifest(MANIFEST_PATH, LOCK_PATH) combined_repos = load_combined_repos(path) @@ -126,7 +123,6 @@ def update_combined(path: Path) -> List[str]: os.makedirs(repos_path, exist_ok=True) updated_repos = [] - notifications = [] for repo in manifest.repos: combined_repo = None @@ -134,17 +130,13 @@ def update_combined(path: Path) -> List[str]: combined_repo = combined_repos[repo.name] del combined_repos[repo.name] try: - new_repo, notification = update_combined_repo( - combined_repo, repo, repos_path - ) + new_repo = update_combined_repo(combined_repo, repo, repos_path) except Exception: logger.exception(f"Failed to updated repository {repo.name}") continue if new_repo is not None: updated_repos.append(new_repo) - if notification is not None: - notifications.append(notification) for combined_repo in combined_repos.values(): remove_repo(combined_repo, path) @@ -156,8 +148,6 @@ def update_combined(path: Path) -> List[str]: with chdir(path): commit_files(["repos.json", "repos.json.lock"], "update repos.json + lock") - return notifications - def setup_combined() -> None: manifest_path = "repos.json" @@ -184,10 +174,4 @@ def combine_command(args: Namespace) -> None: with chdir(combined_path): setup_combined() - notifications = update_combined(combined_path) - - if args.irc_notify: - try: - send(args.irc_notify, notifications) - except Exception as e: - print(f"failed to send irc notifications: {e}") + update_combined(combined_path) diff --git a/nur/error.py b/ci/nur/error.py similarity index 100% rename from nur/error.py rename to ci/nur/error.py diff --git a/nur/fileutils.py b/ci/nur/fileutils.py similarity index 100% rename from nur/fileutils.py rename to ci/nur/fileutils.py diff --git a/nur/format_manifest.py b/ci/nur/format_manifest.py similarity index 100% rename from nur/format_manifest.py rename to ci/nur/format_manifest.py diff --git a/nur/index.py b/ci/nur/index.py similarity index 100% rename from nur/index.py rename to ci/nur/index.py diff --git a/nur/manifest.py b/ci/nur/manifest.py similarity index 100% rename from nur/manifest.py rename to ci/nur/manifest.py diff --git a/nur/path.py b/ci/nur/path.py similarity index 100% rename from nur/path.py rename to ci/nur/path.py diff --git a/nur/prefetch.py b/ci/nur/prefetch.py similarity index 100% rename from nur/prefetch.py rename to ci/nur/prefetch.py diff --git a/nur/update.py b/ci/nur/update.py similarity index 100% rename from nur/update.py rename to ci/nur/update.py diff --git a/pyproject.toml b/ci/pyproject.toml similarity index 100% rename from pyproject.toml rename to ci/pyproject.toml diff --git a/setup.cfg b/ci/setup.cfg similarity index 100% rename from setup.cfg rename to ci/setup.cfg diff --git a/setup.py b/ci/setup.py similarity index 88% rename from setup.py rename to ci/setup.py index 158ee40f9..4b5ff88c8 100644 --- a/setup.py +++ b/ci/setup.py @@ -15,8 +15,7 @@ setup( packages=find_packages(), entry_points={"console_scripts": ["nur = nur:main"]}, extras_require={ - "dev": ["mypy", "flake8>=3.5,<3.6", "black"], - "irc-notifications": ["irc"], + "dev": ["mypy", "flake8", "black"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/ci/test.sh b/ci/test.sh index 6ad9e3655..1dd66332d 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -1,16 +1,11 @@ #!/usr/bin/env nix-shell -#!nix-shell -p bash -i bash -p python3Packages.mypy -p python3Packages.black -p python3Packages.flake8 -p nix +#!nix-shell -p bash -i bash -p python3Packages.mypy -p python3Packages.black -p python3Packages.flake8 -p nixUnstable set -eux -o pipefail # Exit with nonzero exit code if anything fails -nix run '(import ./release.nix {})' -c nur format-manifest -if [ -n "$(git diff --exit-code repos.json)" ]; then - echo "repos.json was not formatted before committing repos.json:" >&2 - git diff --exit-code repos.json - echo "Please run ./bin/nur/format-manifest.py and updates repos.json accordingly!" >&2 - exit 1 -fi +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +cd "${DIR}" # Type checker mypy nur # Format checker @@ -18,5 +13,14 @@ black --check . # Linter flake8 . -nix run '(import ./release.nix {})' -c nur update +cd "${DIR}/.." +nix run "${DIR}#" -- format-manifest +if [ -n "$(git diff --exit-code repos.json)" ]; then + echo "repos.json was not formatted before committing repos.json:" >&2 + git diff --exit-code repos.json + echo "Please run ./bin/nur/format-manifest.py and updates repos.json accordingly!" >&2 + exit 1 +fi + +nix run "${DIR}#" -- update nix-build diff --git a/ci/update-nur-search.sh b/ci/update-nur-search.sh index 75eba0e29..8405982fc 100755 --- a/ci/update-nur-search.sh +++ b/ci/update-nur-search.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -p git -p nix -p bash -i bash +#!nix-shell -p git -p nixUnstable -p bash -i bash set -eu -o pipefail # Exit with nonzero exit code if anything fails @@ -12,13 +12,15 @@ set -x # build package.json for nur-search # --------------------------------- -nix-build --quiet release.nix +nix build "${DIR}#" -git clone --single-branch "https://$API_TOKEN_GITHUB@github.com/nix-community/nur-combined.git" +cd "${DIR}/.." -git clone --recurse-submodules "https://$API_TOKEN_GITHUB@github.com/nix-community/nur-search.git" +git clone --single-branch "https://${API_TOKEN_GITHUB:-git}@github.com/nix-community/nur-combined.git" || git -C nur-combined pull -nix run '(import ./release.nix {})' -c nur index nur-combined > nur-search/data/packages.json +git clone --recurse-submodules "https://${API_TOKEN_GITHUB:-git}@github.com/nix-community/nur-search.git" || git -C nur-search pull + +nix run "${DIR}#" -- index nur-combined > nur-search/data/packages.json # rebuild and publish nur-search repository # ----------------------------------------- diff --git a/ci/update-nur.sh b/ci/update-nur.sh index dda8af0a7..b94344817 100755 --- a/ci/update-nur.sh +++ b/ci/update-nur.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -p git -p nix -p bash -i bash +#!nix-shell -p git -p nixUnstable -p bash -i bash set -eu -o pipefail # Exit with nonzero exit code if anything fails @@ -9,17 +9,14 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/lib/setup-git.sh set -x -nix run '(import ./release.nix {})' -c nur update -nix-build +nix run ${DIR}# -- update + +cd ${DIR}/.. git clone \ --single-branch \ "https://$API_TOKEN_GITHUB@github.com/nix-community/nur-combined.git" -nix run '(import ./release.nix {})' -c nur combine \ - --irc-notify nur-bot@chat.freenode.net:6697/nixos-nur \ - nur-combined - if [[ -z "$(git diff --exit-code)" ]]; then echo "No changes to the output on this push; exiting." else diff --git a/nur/irc_notify.py b/nur/irc_notify.py deleted file mode 100644 index efded4802..000000000 --- a/nur/irc_notify.py +++ /dev/null @@ -1,75 +0,0 @@ -import socket -import ssl -from typing import List, Optional -from urllib.parse import urlparse - - -def notify_irc( - server: str, - nick: str, - password: Optional[str], - channel: str, - tls: bool = True, - port: int = 6697, - messages: List[str] = [], -) -> None: - if not messages: - return - - sock = socket.socket() - if tls: - sock = ssl.wrap_socket( - sock, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_TLSv1_2 - ) - - def _send(command: str) -> int: - return sock.send((f"{command}\r\n").encode()) - - sock.connect((server, port)) - if password: - _send(f"PASS {password}") - _send(f"NICK {nick}") - _send(f"USER {nick} {server} bla :{nick}") - _send(f"JOIN :{channel}") - - for m in messages: - _send(f"PRIVMSG {channel} :{m}") - - _send("INFO") - - while True: - data = sock.recv(4096) - if not data: - raise RuntimeError("Received empty data") - - # Assume INFO reply means we are done - if b"End of /INFO list" in data: - break - - if data.startswith(b"PING"): - sock.send(data.replace(b"PING", b"PONG")) - - sock.send(b"QUIT") - sock.close() - - -def send(url: str, notifications: List[str]) -> None: - parsed = urlparse(f"http://{url}") - username = parsed.username or "nur-bot" - server = parsed.hostname or "chat.freenode.de" - if parsed.path != "/" or parsed.path == "": - channel = f"#{parsed.path[1:]}" - else: - channel = "#nixos-nur" - port = parsed.port or 6697 - password = parsed.password - if len(notifications) == 0: - return - notify_irc( - server=server, - nick=username, - password=password, - channel=channel, - port=port, - messages=notifications, - ) diff --git a/release.nix b/release.nix deleted file mode 100644 index 5170a8a62..000000000 --- a/release.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ nixpkgs ? import }: - -with import {}; - -python3Packages.buildPythonApplication { - name = "nur"; - src = ./.; - - doCheck = true; - - makeWrapperArgs = [ - "--prefix" "PATH" ":" "${pkgs.lib.makeBinPath [ nix-prefetch-git git nix ]}" - "--set" "LOCALE_ARCHIVE" "${glibcLocales}/lib/locale/locale-archive" - ]; -}