From 38cfeb3fbd6bde46a2f8a41647d8aab48894e207 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Mon, 11 Jan 2021 23:20:26 +0100 Subject: [PATCH 01/13] added the attributes improvement back again --- nur/index.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nur/index.py b/nur/index.py index 92bc97a6b..4e87ee9e6 100644 --- a/nur/index.py +++ b/nur/index.py @@ -42,7 +42,22 @@ callPackage (nur.repo-sources."%s" + "/%s") {} if position is not None and position.startswith("/nix/store"): path_str, line = position.rsplit(":", 1) path = Path(path_str) + # I've decided to just take these 2 repositories, + # update this whenever someone decided to use a recipe source other than + # NUR or nixpkgs to override packages on. right now this is about as accurate as + # `nix edit` is + # TODO find commit hash + prefixes = { + "nixpkgs": "https://github.com/nixos/nixpkgs/tree/master/", + "nur": "https://github.com/nix-community/nur-combined/tree/master/", + } stripped = path.parts[4:] + attrPath = "/".join(stripped[1:]) + location = f"{prefixes[stripped[0]]}{attrPath}" + pkg["meta"]["position"] = f"{location}#L{line}" + elif position is not None and position.find("nur-combined") > -1: + path_str, line = position.rsplit(":", 1) + stripped = path_str.partition(f"nur-combined/repos/{repo}")[2] pkg["meta"]["position"] = f"{prefix}{stripped}#L{line}" else: pkg["meta"]["position"] = prefix From e28963ae61127bd08414217d92b274b6eff4dec9 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Mon, 11 Jan 2021 23:33:44 +0100 Subject: [PATCH 02/13] fixed some edge case that broke nur search --- nur/index.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nur/index.py b/nur/index.py index 4e87ee9e6..0254baeab 100644 --- a/nur/index.py +++ b/nur/index.py @@ -52,9 +52,13 @@ callPackage (nur.repo-sources."%s" + "/%s") {} "nur": "https://github.com/nix-community/nur-combined/tree/master/", } stripped = path.parts[4:] - attrPath = "/".join(stripped[1:]) - location = f"{prefixes[stripped[0]]}{attrPath}" - pkg["meta"]["position"] = f"{location}#L{line}" + if stripped[0] not in prefixes: + print(f"we could not find {stripped} , you can file an issue at https://github.com/nix-community/NUR/issues to the indexing file if you think this is a mistake", file=sys.stderr) + pkg["meta"]["position"] = prefix + else: + attrPath = "/".join(stripped[1:]) + location = f"{prefixes[stripped[0]]}{attrPath}" + pkg["meta"]["position"] = f"{location}#L{line}" elif position is not None and position.find("nur-combined") > -1: path_str, line = position.rsplit(":", 1) stripped = path_str.partition(f"nur-combined/repos/{repo}")[2] From 830d222641bc70840d839140bfe45bdfd7c3a9cf Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Mon, 11 Jan 2021 23:41:08 +0100 Subject: [PATCH 03/13] python black reformat --- nur/index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nur/index.py b/nur/index.py index 0254baeab..c695ef931 100644 --- a/nur/index.py +++ b/nur/index.py @@ -53,7 +53,10 @@ callPackage (nur.repo-sources."%s" + "/%s") {} } stripped = path.parts[4:] if stripped[0] not in prefixes: - print(f"we could not find {stripped} , you can file an issue at https://github.com/nix-community/NUR/issues to the indexing file if you think this is a mistake", file=sys.stderr) + print( + f"we could not find {stripped} , you can file an issue at https://github.com/nix-community/NUR/issues to the indexing file if you think this is a mistake", + file=sys.stderr, + ) pkg["meta"]["position"] = prefix else: attrPath = "/".join(stripped[1:]) From 0493dbe7939d031d8bbc590c985ff9aa78ebaa3f Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 00:17:16 +0100 Subject: [PATCH 04/13] added URL to the list of arguments There might be an issue in CI where the nur-combined directory does not get cloned. To remedy this, we pass the url of a repos.json element to the indexing function, with as goal resolving the "pkgs" prefix, as it seemed to come up quite often. There are some other prefixes that I haven't figured out yet, but at least this is a start --- nur/index.py | 6 ++++-- temp.nix | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 temp.nix diff --git a/nur/index.py b/nur/index.py index c695ef931..49a21b80f 100644 --- a/nur/index.py +++ b/nur/index.py @@ -7,7 +7,7 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict -def index_repo(directory: Path, repo: str, expression_file: str) -> Dict[str, Any]: +def index_repo(directory: Path, repo: str, expression_file: str, url: str) -> Dict[str, Any]: default_nix = directory.joinpath("default.nix").resolve() expr = """ with import {}; @@ -50,6 +50,8 @@ callPackage (nur.repo-sources."%s" + "/%s") {} prefixes = { "nixpkgs": "https://github.com/nixos/nixpkgs/tree/master/", "nur": "https://github.com/nix-community/nur-combined/tree/master/", + # this tends to come up when nur-combined isn't indexed yet + "pkgs": url + "/", } stripped = path.parts[4:] if stripped[0] not in prefixes: @@ -81,7 +83,7 @@ def index_command(args: Namespace) -> None: pkgs: Dict[str, Any] = {} for (repo, data) in repos.items(): - repo_pkgs = index_repo(directory, repo, data.get("file", "default.nix")) + repo_pkgs = index_repo(directory, repo, data.get("file", "default.nix"), data.get("url", "https://github.com/nixos/nixpkgs")) pkgs.update(repo_pkgs) json.dump(pkgs, sys.stdout, indent=4) diff --git a/temp.nix b/temp.nix new file mode 100644 index 000000000..4e1192e67 --- /dev/null +++ b/temp.nix @@ -0,0 +1,5 @@ +with import {}; +let + nur = import ./. { inherit pkgs;}; +in + callPackage (nur.repo-sources.crazazy + "/pkgs/default.nix") From d082944dab6084f3b208f3d5de3334744a7a485e Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 00:55:00 +0100 Subject: [PATCH 05/13] Tried to fix a bunch of missing attr paths I'm not sure if this is genuinely the case, but fro the CI logs it seems to me that nur-combined does not actually get used when evaluating repositories, which was an assumption that I previously had. The current addition tries it's best to resolve the issue of default.nix not refering to a local path, but to a derivation in the nix store. Currently, only gitlab and github links are supported, as smaller git hosts do not get special treatment from nixpkgs, and thus their derivation name is not "source" --- nur/index.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nur/index.py b/nur/index.py index 49a21b80f..66c559cfa 100644 --- a/nur/index.py +++ b/nur/index.py @@ -50,11 +50,21 @@ callPackage (nur.repo-sources."%s" + "/%s") {} prefixes = { "nixpkgs": "https://github.com/nixos/nixpkgs/tree/master/", "nur": "https://github.com/nix-community/nur-combined/tree/master/", - # this tends to come up when nur-combined isn't indexed yet - "pkgs": url + "/", } stripped = path.parts[4:] - if stripped[0] not in prefixes: + if path.parts[3].endswith("source"): + def url_contains(host: str) -> bool: + return url.find(host) != -1 + canonical_url = url + if url_contains("github"): + canonical_url += "/blob" + elif url_contains("gitlab"): + canonical_url += "/-/blob" + attrPath = "/".join(stripped) + location = f"{canonical_url}{attrPath}" + pkg["meta"]["position"] = f"{location}#L{line}" + elif stripped[0] not in prefixes: + print(path, file=sys.stderr) print( f"we could not find {stripped} , you can file an issue at https://github.com/nix-community/NUR/issues to the indexing file if you think this is a mistake", file=sys.stderr, From 684a44f66fab7ed73b0aa14e966857a1de36ae46 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 00:59:11 +0100 Subject: [PATCH 06/13] forgot black reformatting again --- nur/index.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nur/index.py b/nur/index.py index 66c559cfa..b5969ef43 100644 --- a/nur/index.py +++ b/nur/index.py @@ -7,7 +7,9 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict -def index_repo(directory: Path, repo: str, expression_file: str, url: str) -> Dict[str, Any]: +def index_repo( + directory: Path, repo: str, expression_file: str, url: str +) -> Dict[str, Any]: default_nix = directory.joinpath("default.nix").resolve() expr = """ with import {}; @@ -53,8 +55,10 @@ callPackage (nur.repo-sources."%s" + "/%s") {} } stripped = path.parts[4:] if path.parts[3].endswith("source"): + def url_contains(host: str) -> bool: return url.find(host) != -1 + canonical_url = url if url_contains("github"): canonical_url += "/blob" @@ -93,7 +97,12 @@ def index_command(args: Namespace) -> None: pkgs: Dict[str, Any] = {} for (repo, data) in repos.items(): - repo_pkgs = index_repo(directory, repo, data.get("file", "default.nix"), data.get("url", "https://github.com/nixos/nixpkgs")) + repo_pkgs = index_repo( + directory, + repo, + data.get("file", "default.nix"), + data.get("url", "https://github.com/nixos/nixpkgs"), + ) pkgs.update(repo_pkgs) json.dump(pkgs, sys.stdout, indent=4) From b02737f18637a6e6f481573ce9590b9e49315907 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 01:00:56 +0100 Subject: [PATCH 07/13] forgot a trailing slash as well --- nur/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nur/index.py b/nur/index.py index b5969ef43..e2f26196a 100644 --- a/nur/index.py +++ b/nur/index.py @@ -61,9 +61,9 @@ callPackage (nur.repo-sources."%s" + "/%s") {} canonical_url = url if url_contains("github"): - canonical_url += "/blob" + canonical_url += "/blob/" elif url_contains("gitlab"): - canonical_url += "/-/blob" + canonical_url += "/-/blob/" attrPath = "/".join(stripped) location = f"{canonical_url}{attrPath}" pkg["meta"]["position"] = f"{location}#L{line}" From 157d196244f46fac791d8785e36a68547aa31003 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 10:49:16 +0100 Subject: [PATCH 08/13] separated attribute logic into it's own function --- nur/index.py | 97 +++++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/nur/index.py b/nur/index.py index e2f26196a..8fd5f3140 100644 --- a/nur/index.py +++ b/nur/index.py @@ -7,6 +7,55 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict +def resolve_source(pkg: Dict, repo: str, url: str) -> str: + # TODO commit hash + prefix = f"https://github.com/nix-community/nur-combined/tree/master/repos/{repo}" + position = pkg["meta"].get("position", None) + if position is not None and position.startswith("/nix/store"): + path_str, line = position.rsplit(":", 1) + path = Path(path_str) + # I've decided to just take these 2 repositories, + # update this whenever someone decided to use a recipe source other than + # NUR or nixpkgs to override packages on. right now this is about as accurate as + # `nix edit` is + # TODO find commit hash + prefixes = { + "nixpkgs": "https://github.com/nixos/nixpkgs/tree/master/", + "nur": "https://github.com/nix-community/nur-combined/tree/master/", + } + stripped = path.parts[4:] + if path.parts[3].endswith("source"): + + def url_contains(host: str) -> bool: + return url.find(host) != -1 + + canonical_url = url + if url_contains("github"): + canonical_url += "/blob/" + elif url_contains("gitlab"): + canonical_url += "/-/blob/" + attrPath = "/".join(stripped) + location = f"{canonical_url}{attrPath}" + return f"{location}#L{line}" + elif stripped[0] not in prefixes: + print(path, file=sys.stderr) + print( + f"we could not find {stripped} , you can file an issue at https://github.com/nix-community/NUR/issues to the indexing file if you think this is a mistake", + file=sys.stderr, + ) + return prefix + else: + attrPath = "/".join(stripped[1:]) + location = f"{prefixes[stripped[0]]}{attrPath}" + return f"{location}#L{line}" + elif position is not None and position.find("nur-combined") > -1: + path_str, line = position.rsplit(":", 1) + stripped = path_str.partition(f"nur-combined/repos/{repo}")[2] + return f"{prefix}{stripped}#L{line}" + else: + return prefix + + def index_repo( directory: Path, repo: str, expression_file: str, url: str ) -> Dict[str, Any]: @@ -38,53 +87,9 @@ callPackage (nur.repo-sources."%s" + "/%s") {} for name, pkg in raw_pkgs.items(): pkg["_attr"] = name pkg["_repo"] = repo - position = pkg["meta"].get("position", None) - # TODO commit hash - prefix = f"https://github.com/nix-community/nur-combined/tree/master/repos/{repo}" - if position is not None and position.startswith("/nix/store"): - path_str, line = position.rsplit(":", 1) - path = Path(path_str) - # I've decided to just take these 2 repositories, - # update this whenever someone decided to use a recipe source other than - # NUR or nixpkgs to override packages on. right now this is about as accurate as - # `nix edit` is - # TODO find commit hash - prefixes = { - "nixpkgs": "https://github.com/nixos/nixpkgs/tree/master/", - "nur": "https://github.com/nix-community/nur-combined/tree/master/", - } - stripped = path.parts[4:] - if path.parts[3].endswith("source"): - - def url_contains(host: str) -> bool: - return url.find(host) != -1 - - canonical_url = url - if url_contains("github"): - canonical_url += "/blob/" - elif url_contains("gitlab"): - canonical_url += "/-/blob/" - attrPath = "/".join(stripped) - location = f"{canonical_url}{attrPath}" - pkg["meta"]["position"] = f"{location}#L{line}" - elif stripped[0] not in prefixes: - print(path, file=sys.stderr) - print( - f"we could not find {stripped} , you can file an issue at https://github.com/nix-community/NUR/issues to the indexing file if you think this is a mistake", - file=sys.stderr, - ) - pkg["meta"]["position"] = prefix - else: - attrPath = "/".join(stripped[1:]) - location = f"{prefixes[stripped[0]]}{attrPath}" - pkg["meta"]["position"] = f"{location}#L{line}" - elif position is not None and position.find("nur-combined") > -1: - path_str, line = position.rsplit(":", 1) - stripped = path_str.partition(f"nur-combined/repos/{repo}")[2] - pkg["meta"]["position"] = f"{prefix}{stripped}#L{line}" - else: - pkg["meta"]["position"] = prefix + pkg["meta"]["position"] = resolve_source(pkg, repo, url) pkgs[f"nur.repos.{repo}.{name}"] = pkg + return pkgs From 888312333b5ca498e2fea927ec987352ba9282fd Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 10:52:26 +0100 Subject: [PATCH 09/13] forgot to add branch names --- nur/index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nur/index.py b/nur/index.py index 8fd5f3140..241da1938 100644 --- a/nur/index.py +++ b/nur/index.py @@ -30,10 +30,11 @@ def resolve_source(pkg: Dict, repo: str, url: str) -> str: return url.find(host) != -1 canonical_url = url + # if we want to add the option of specifying branches, we have to update this if url_contains("github"): - canonical_url += "/blob/" + canonical_url += "/blob/master/" elif url_contains("gitlab"): - canonical_url += "/-/blob/" + canonical_url += "/-/blob/master/" attrPath = "/".join(stripped) location = f"{canonical_url}{attrPath}" return f"{location}#L{line}" From 023c82475c797338b4c8a9f814b2e40668015a68 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 11:07:01 +0100 Subject: [PATCH 10/13] Replace url_contains function with the 'in' operator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- nur/index.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/nur/index.py b/nur/index.py index 241da1938..c27947cd6 100644 --- a/nur/index.py +++ b/nur/index.py @@ -26,14 +26,11 @@ def resolve_source(pkg: Dict, repo: str, url: str) -> str: stripped = path.parts[4:] if path.parts[3].endswith("source"): - def url_contains(host: str) -> bool: - return url.find(host) != -1 - canonical_url = url # if we want to add the option of specifying branches, we have to update this - if url_contains("github"): + if "github" in url: canonical_url += "/blob/master/" - elif url_contains("gitlab"): + elif "gitlab" in url: canonical_url += "/-/blob/master/" attrPath = "/".join(stripped) location = f"{canonical_url}{attrPath}" @@ -49,7 +46,7 @@ def resolve_source(pkg: Dict, repo: str, url: str) -> str: attrPath = "/".join(stripped[1:]) location = f"{prefixes[stripped[0]]}{attrPath}" return f"{location}#L{line}" - elif position is not None and position.find("nur-combined") > -1: + elif position is not None and "nur-combined" in position: path_str, line = position.rsplit(":", 1) stripped = path_str.partition(f"nur-combined/repos/{repo}")[2] return f"{prefix}{stripped}#L{line}" From 53c6bc3eccd1eeee4a93e0bc5f17047c1dc31e25 Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Tue, 12 Jan 2021 11:40:20 +0100 Subject: [PATCH 11/13] removed temp.nix file --- temp.nix | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 temp.nix diff --git a/temp.nix b/temp.nix deleted file mode 100644 index 4e1192e67..000000000 --- a/temp.nix +++ /dev/null @@ -1,5 +0,0 @@ -with import {}; -let - nur = import ./. { inherit pkgs;}; -in - callPackage (nur.repo-sources.crazazy + "/pkgs/default.nix") From 239a90d5a63dc1eb08e3bb1a26e5c9792d0fda60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 13 Jan 2021 08:28:41 +0100 Subject: [PATCH 12/13] use lower case variables --- nur/index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nur/index.py b/nur/index.py index c27947cd6..4b95f50b7 100644 --- a/nur/index.py +++ b/nur/index.py @@ -32,8 +32,8 @@ def resolve_source(pkg: Dict, repo: str, url: str) -> str: canonical_url += "/blob/master/" elif "gitlab" in url: canonical_url += "/-/blob/master/" - attrPath = "/".join(stripped) - location = f"{canonical_url}{attrPath}" + attr_path = "/".join(stripped) + location = f"{canonical_url}{attr_path}" return f"{location}#L{line}" elif stripped[0] not in prefixes: print(path, file=sys.stderr) @@ -43,8 +43,8 @@ def resolve_source(pkg: Dict, repo: str, url: str) -> str: ) return prefix else: - attrPath = "/".join(stripped[1:]) - location = f"{prefixes[stripped[0]]}{attrPath}" + attr_path = "/".join(stripped[1:]) + location = f"{prefixes[stripped[0]]}{attr_path}" return f"{location}#L{line}" elif position is not None and "nur-combined" in position: path_str, line = position.rsplit(":", 1) From 20d4e9bd91eab0427714552b55206cc450af766b Mon Sep 17 00:00:00 2001 From: Erik Oosting Date: Wed, 13 Jan 2021 11:42:54 +0100 Subject: [PATCH 13/13] Switch from choosing the master branch to choosing the default branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- nur/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nur/index.py b/nur/index.py index 4b95f50b7..3ed20dc4d 100644 --- a/nur/index.py +++ b/nur/index.py @@ -29,9 +29,9 @@ def resolve_source(pkg: Dict, repo: str, url: str) -> str: canonical_url = url # if we want to add the option of specifying branches, we have to update this if "github" in url: - canonical_url += "/blob/master/" + canonical_url += "/blob/HEAD/" elif "gitlab" in url: - canonical_url += "/-/blob/master/" + canonical_url += "/-/blob/HEAD/" attr_path = "/".join(stripped) location = f"{canonical_url}{attr_path}" return f"{location}#L{line}"