diff --git a/clusters/github-runner/README.md b/clusters/github-runner/README.md deleted file mode 100644 index ade6605..0000000 --- a/clusters/github-runner/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Self-Hosted CI on Macbook Pro - -**WARNING: WIP Notes**. Expect final version in nixos.asia as a blog post. - -## Approach - -While we could use the `nix-darwin` module for Github Runners, we do it the other way. First, create a aarch64-linux NixOS VM (I use Parallels Desktop) and do everything there. Then, setup distributed builds to have the VM do aarch64-darwin builds remotely on the host machine (the Macbook Pro). The former is done buy `./nixos-module.nix`, while the latter is done by `./darwin-module.nix`. - -### Facts - -- I use 1Password (managed by colmena secrets) to store the GitHUb classic PAT. -- On macOS, go to Remote Login and allow SSH access for the `github-runner` user, or allow for all users; otherwise our Linux VM won't be able to remote build on the Mac. - - The Linux VM's `/etc/ssh/ssh_host_ed25519_key` is used to authorize itself to connect to the Mac. -- The author has observed the official "linux-builder" to be slow, in comparison to a Parallels VM. Prefer setting up a Parallels VM if you can. - -## Known Issues - -- GitHub token must be provided to avoid the "API rate limit exceeded" error (which can happen if you do all this on your laptop and work around the world). See https://github.com/srid/nixos-config/issues/54 -- GitHub runner might crash due to out of sync time on the VM. If you are Parallels, you should [sync time from Mac](https://kb.parallels.com/113271). - -## Usage - -- `/systems/darwin.nix` (macOS config) -- `/systems/github-runner.nix` (NixOS Linux VM config) - -## See also - -- [Zulip notes](https://nixos.zulipchat.com/#narrow/stream/413948-nixos/topic/Self-hosted.20GitHub.20runners) -- Old architecture, of running the runners on macOS: https://github.com/srid/nixos-config/tree/bdf0ad9619abe9785097981dfec4fb6f3d9f1b5d/nix-darwin/ci diff --git a/clusters/github-runner/darwin-module.nix b/clusters/github-runner/darwin-module.nix deleted file mode 100644 index 692c3de..0000000 --- a/clusters/github-runner/darwin-module.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ pkgs, ... }: - -{ - # Github runner CI - users = { - knownUsers = [ "github-runner" ]; - forceRecreate = true; - users.github-runner = { - uid = 1009; - description = "GitHub Runner"; - home = "/Users/github-runner"; - createHome = true; - shell = pkgs.bashInteractive; - # NOTE: Go to macOS Remote-Login settings and allow all users to ssh. - openssh.authorizedKeys.keys = [ - # github-runner VM's /etc/ssh/ssh_host_ed25519_key.pub - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJUJvyuUnIs2q2TkJq29wqJ6HyOAeMmIK8PcH7xAlpVY root@github-runner" - ]; - }; - }; - nix.settings.trusted-users = [ "github-runner" ]; -} diff --git a/clusters/github-runner/nixos-module.nix b/clusters/github-runner/nixos-module.nix deleted file mode 100644 index 67b10a4..0000000 --- a/clusters/github-runner/nixos-module.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ flake, pkgs, lib, ... }: - -let - inherit (flake) inputs; - inherit (inputs) self; - user = "github-runner"; - group = "github-runner"; - tokenFile = "/run/keys/github-runner-token.secret"; # See colmena keys in top-level flake.nix - repos = import ./repos.nix; - runner-pkgs = (import ./runner-pkgs.nix { inherit pkgs lib; }); - # Convenient function to create multiple runners per single personal repo. - mkPersonalRunners = user: - lib.concatMapAttrs (repoName: meta: - lib.listToAttrs (lib.flip builtins.map (lib.range 1 meta.num) (idx: - let - name = "${repoName}-${builtins.toString idx}"; - in - lib.nameValuePair name { - inherit user group tokenFile name; - enable = true; - replace = true; - ephemeral = true; - extraPackages = with pkgs; runner-pkgs ++ [ - # Standard nix tools - nixci - cachix - # For nixos-flake - sd - ]; - url = "https://github.com/${user}/${repoName}"; - }))); - hostIP = "10.37.129.2"; # Find out using `ifconfig` on host, looking for bridge101 -in -{ - # User - users.users.${user} = { - inherit group; - isSystemUser = true; - }; - users.groups.${group} = { }; - nix.settings.trusted-users = [ user ]; - - # No way to do this: https://github.com/NixOS/nix/issues/6536 - #nix.extraOptions = '' - # !include /run/keys/nix-conf-gh-token.secret - #''; - - # Runners - services.github-runners = mkPersonalRunners "srid" repos.srid; - - # macOS remote builder - nix.distributedBuilds = true; - nix.buildMachines = [{ - hostName = hostIP; - systems = [ "aarch64-darwin" "x86_64-darwin" ]; - maxJobs = 6; # 6 cores - protocol = "ssh-ng"; - sshUser = user; - sshKey = "/etc/ssh/ssh_host_ed25519_key"; - }]; -} diff --git a/clusters/github-runner/repos.nix b/clusters/github-runner/repos.nix deleted file mode 100644 index a6388fd..0000000 --- a/clusters/github-runner/repos.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # My personal repos with self-hosted runners: - # `num` is usually based on matrix configuration size - srid = { - nixos-config.num = 2; - # TODO: Moving these to Juspay - # nixci.num = 2; - # haskell-flake.num = 2 * 7; - # nixos-flake.num = 2 * 5; - # emanote.num = 2; - }; -} diff --git a/clusters/github-runner/runner-pkgs.nix b/clusters/github-runner/runner-pkgs.nix deleted file mode 100644 index 3b2616a..0000000 --- a/clusters/github-runner/runner-pkgs.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ pkgs, lib, ... }: - -# Tools already available in standard GitHub Runners; so we provide -# them here: -with pkgs; [ - coreutils - which - jq - # https://github.com/actions/upload-pages-artifact/blob/56afc609e74202658d3ffba0e8f6dda462b719fa/action.yml#L40 - (runCommandNoCC "gtar" { } '' - mkdir -p $out/bin - ln -s ${lib.getExe gnutar} $out/bin/gtar - '') -] diff --git a/flake.nix b/flake.nix index 5dfaf49..1487eb1 100644 --- a/flake.nix +++ b/flake.nix @@ -51,11 +51,6 @@ self.nixos-flake.lib.mkMacosSystem ./systems/darwin.nix; - # Configuration for a NixOS VM (running on my Mac) - nixosConfigurations.github-runner = - self.nixos-flake.lib.mkLinuxSystem - ./systems/github-runner.nix; - # Hetzner dedicated nixosConfigurations.immediacy = self.nixos-flake.lib.mkLinuxSystem diff --git a/justfile b/justfile index d9ab975..1f5681a 100644 --- a/justfile +++ b/justfile @@ -9,26 +9,3 @@ activate: # Format the nix source tree fmt: treefmt - -# Deploy to all remote machines -deploy: - colmena apply --build-on-target - -# Deploy to github-runner VM -[group('github-runner')] -gr-deploy: - colmena apply --build-on-target --on github-runner - -# Re-animate the VM that was suspended until now. -[group('github-runner')] -gr-animate: - colmena upload-keys - ssh -t github-runner "sudo systemctl restart --all github-runner-*" - -[group('github-runner')] -gr-inspect: - ssh -t github-runner "sudo systemctl status --all github-runner-*" - -[group('github-runner')] -gr-ssh: - ssh -t github-runner \ No newline at end of file diff --git a/systems/darwin.nix b/systems/darwin.nix index c721e3f..497b09d 100644 --- a/systems/darwin.nix +++ b/systems/darwin.nix @@ -8,7 +8,6 @@ in imports = [ self.darwinModules.default "${self}/nix-darwin/zsh-completion-fix.nix" - "${self}/clusters/github-runner/darwin-module.nix" ]; nixpkgs.hostPlatform = "aarch64-darwin"; diff --git a/systems/github-runner.nix b/systems/github-runner.nix deleted file mode 100644 index 4b23862..0000000 --- a/systems/github-runner.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ flake, ... }: - -let - inherit (flake) inputs; - inherit (inputs) self; -in -{ - imports = [ - inputs.disko.nixosModules.disko - "${self}/nixos/disko/trivial.nix" - "${self}/nixos/parallels-vm.nix" - "${self}/nixos/nix.nix" - "${self}/nixos/current-location.nix" - "${self}/nixos/self/primary-as-admin.nix" - "${self}/nixos/server/harden/basics.nix" - "${self}/clusters/github-runner/nixos-module.nix" - ]; - - system.stateVersion = "23.11"; - networking.hostName = "github-runner"; - nixpkgs.hostPlatform = "aarch64-linux"; - boot = { - binfmt.emulatedSystems = [ "x86_64-linux" ]; - loader = { - systemd-boot.enable = true; - efi.canTouchEfiVariables = true; - }; - }; - services.openssh.enable = true; -}