mirror of
https://github.com/srid/nixos-config.git
synced 2026-01-10 02:02:36 +08:00
Remove clusters/github-runner
New module upcoming ...
This commit is contained in:
Sridhar Ratnakumar
Sridhar Ratnakumar
parent
06cea073ec
commit
4d29a310ea
9 changed files with 0 additions and 197 deletions
|
|
@ -1,29 +0,0 @@
|
|||
# Self-Hosted CI on Macbook Pro
|
||||
|
||||
**WARNING: WIP Notes**. Expect final version in nixos.asia as a blog post.
|
||||
|
||||
## Approach
|
||||
|
||||
While we could use the `nix-darwin` module for Github Runners, we do it the other way. First, create a aarch64-linux NixOS VM (I use Parallels Desktop) and do everything there. Then, setup distributed builds to have the VM do aarch64-darwin builds remotely on the host machine (the Macbook Pro). The former is done buy `./nixos-module.nix`, while the latter is done by `./darwin-module.nix`.
|
||||
|
||||
### Facts
|
||||
|
||||
- I use 1Password (managed by colmena secrets) to store the GitHUb classic PAT.
|
||||
- On macOS, go to Remote Login and allow SSH access for the `github-runner` user, or allow for all users; otherwise our Linux VM won't be able to remote build on the Mac.
|
||||
- The Linux VM's `/etc/ssh/ssh_host_ed25519_key` is used to authorize itself to connect to the Mac.
|
||||
- The author has observed the official "linux-builder" to be slow, in comparison to a Parallels VM. Prefer setting up a Parallels VM if you can.
|
||||
|
||||
## Known Issues
|
||||
|
||||
- GitHub token must be provided to avoid the "API rate limit exceeded" error (which can happen if you do all this on your laptop and work around the world). See https://github.com/srid/nixos-config/issues/54
|
||||
- GitHub runner might crash due to out of sync time on the VM. If you are Parallels, you should [sync time from Mac](https://kb.parallels.com/113271).
|
||||
|
||||
## Usage
|
||||
|
||||
- `/systems/darwin.nix` (macOS config)
|
||||
- `/systems/github-runner.nix` (NixOS Linux VM config)
|
||||
|
||||
## See also
|
||||
|
||||
- [Zulip notes](https://nixos.zulipchat.com/#narrow/stream/413948-nixos/topic/Self-hosted.20GitHub.20runners)
|
||||
- Old architecture, of running the runners on macOS: https://github.com/srid/nixos-config/tree/bdf0ad9619abe9785097981dfec4fb6f3d9f1b5d/nix-darwin/ci
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
{ pkgs, ... }:
|
||||
|
||||
{
|
||||
# Github runner CI
|
||||
users = {
|
||||
knownUsers = [ "github-runner" ];
|
||||
forceRecreate = true;
|
||||
users.github-runner = {
|
||||
uid = 1009;
|
||||
description = "GitHub Runner";
|
||||
home = "/Users/github-runner";
|
||||
createHome = true;
|
||||
shell = pkgs.bashInteractive;
|
||||
# NOTE: Go to macOS Remote-Login settings and allow all users to ssh.
|
||||
openssh.authorizedKeys.keys = [
|
||||
# github-runner VM's /etc/ssh/ssh_host_ed25519_key.pub
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJUJvyuUnIs2q2TkJq29wqJ6HyOAeMmIK8PcH7xAlpVY root@github-runner"
|
||||
];
|
||||
};
|
||||
};
|
||||
nix.settings.trusted-users = [ "github-runner" ];
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
{ flake, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (flake) inputs;
|
||||
inherit (inputs) self;
|
||||
user = "github-runner";
|
||||
group = "github-runner";
|
||||
tokenFile = "/run/keys/github-runner-token.secret"; # See colmena keys in top-level flake.nix
|
||||
repos = import ./repos.nix;
|
||||
runner-pkgs = (import ./runner-pkgs.nix { inherit pkgs lib; });
|
||||
# Convenient function to create multiple runners per single personal repo.
|
||||
mkPersonalRunners = user:
|
||||
lib.concatMapAttrs (repoName: meta:
|
||||
lib.listToAttrs (lib.flip builtins.map (lib.range 1 meta.num) (idx:
|
||||
let
|
||||
name = "${repoName}-${builtins.toString idx}";
|
||||
in
|
||||
lib.nameValuePair name {
|
||||
inherit user group tokenFile name;
|
||||
enable = true;
|
||||
replace = true;
|
||||
ephemeral = true;
|
||||
extraPackages = with pkgs; runner-pkgs ++ [
|
||||
# Standard nix tools
|
||||
nixci
|
||||
cachix
|
||||
# For nixos-flake
|
||||
sd
|
||||
];
|
||||
url = "https://github.com/${user}/${repoName}";
|
||||
})));
|
||||
hostIP = "10.37.129.2"; # Find out using `ifconfig` on host, looking for bridge101
|
||||
in
|
||||
{
|
||||
# User
|
||||
users.users.${user} = {
|
||||
inherit group;
|
||||
isSystemUser = true;
|
||||
};
|
||||
users.groups.${group} = { };
|
||||
nix.settings.trusted-users = [ user ];
|
||||
|
||||
# No way to do this: https://github.com/NixOS/nix/issues/6536
|
||||
#nix.extraOptions = ''
|
||||
# !include /run/keys/nix-conf-gh-token.secret
|
||||
#'';
|
||||
|
||||
# Runners
|
||||
services.github-runners = mkPersonalRunners "srid" repos.srid;
|
||||
|
||||
# macOS remote builder
|
||||
nix.distributedBuilds = true;
|
||||
nix.buildMachines = [{
|
||||
hostName = hostIP;
|
||||
systems = [ "aarch64-darwin" "x86_64-darwin" ];
|
||||
maxJobs = 6; # 6 cores
|
||||
protocol = "ssh-ng";
|
||||
sshUser = user;
|
||||
sshKey = "/etc/ssh/ssh_host_ed25519_key";
|
||||
}];
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
{
|
||||
# My personal repos with self-hosted runners:
|
||||
# `num` is usually based on matrix configuration size
|
||||
srid = {
|
||||
nixos-config.num = 2;
|
||||
# TODO: Moving these to Juspay
|
||||
# nixci.num = 2;
|
||||
# haskell-flake.num = 2 * 7;
|
||||
# nixos-flake.num = 2 * 5;
|
||||
# emanote.num = 2;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
{ pkgs, lib, ... }:
|
||||
|
||||
# Tools already available in standard GitHub Runners; so we provide
|
||||
# them here:
|
||||
with pkgs; [
|
||||
coreutils
|
||||
which
|
||||
jq
|
||||
# https://github.com/actions/upload-pages-artifact/blob/56afc609e74202658d3ffba0e8f6dda462b719fa/action.yml#L40
|
||||
(runCommandNoCC "gtar" { } ''
|
||||
mkdir -p $out/bin
|
||||
ln -s ${lib.getExe gnutar} $out/bin/gtar
|
||||
'')
|
||||
]
|
||||
|
|
@ -51,11 +51,6 @@
|
|||
self.nixos-flake.lib.mkMacosSystem
|
||||
./systems/darwin.nix;
|
||||
|
||||
# Configuration for a NixOS VM (running on my Mac)
|
||||
nixosConfigurations.github-runner =
|
||||
self.nixos-flake.lib.mkLinuxSystem
|
||||
./systems/github-runner.nix;
|
||||
|
||||
# Hetzner dedicated
|
||||
nixosConfigurations.immediacy =
|
||||
self.nixos-flake.lib.mkLinuxSystem
|
||||
|
|
|
|||
23
justfile
23
justfile
|
|
@ -9,26 +9,3 @@ activate:
|
|||
# Format the nix source tree
|
||||
fmt:
|
||||
treefmt
|
||||
|
||||
# Deploy to all remote machines
|
||||
deploy:
|
||||
colmena apply --build-on-target
|
||||
|
||||
# Deploy to github-runner VM
|
||||
[group('github-runner')]
|
||||
gr-deploy:
|
||||
colmena apply --build-on-target --on github-runner
|
||||
|
||||
# Re-animate the VM that was suspended until now.
|
||||
[group('github-runner')]
|
||||
gr-animate:
|
||||
colmena upload-keys
|
||||
ssh -t github-runner "sudo systemctl restart --all github-runner-*"
|
||||
|
||||
[group('github-runner')]
|
||||
gr-inspect:
|
||||
ssh -t github-runner "sudo systemctl status --all github-runner-*"
|
||||
|
||||
[group('github-runner')]
|
||||
gr-ssh:
|
||||
ssh -t github-runner
|
||||
|
|
@ -8,7 +8,6 @@ in
|
|||
imports = [
|
||||
self.darwinModules.default
|
||||
"${self}/nix-darwin/zsh-completion-fix.nix"
|
||||
"${self}/clusters/github-runner/darwin-module.nix"
|
||||
];
|
||||
|
||||
nixpkgs.hostPlatform = "aarch64-darwin";
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
{ flake, ... }:
|
||||
|
||||
let
|
||||
inherit (flake) inputs;
|
||||
inherit (inputs) self;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
inputs.disko.nixosModules.disko
|
||||
"${self}/nixos/disko/trivial.nix"
|
||||
"${self}/nixos/parallels-vm.nix"
|
||||
"${self}/nixos/nix.nix"
|
||||
"${self}/nixos/current-location.nix"
|
||||
"${self}/nixos/self/primary-as-admin.nix"
|
||||
"${self}/nixos/server/harden/basics.nix"
|
||||
"${self}/clusters/github-runner/nixos-module.nix"
|
||||
];
|
||||
|
||||
system.stateVersion = "23.11";
|
||||
networking.hostName = "github-runner";
|
||||
nixpkgs.hostPlatform = "aarch64-linux";
|
||||
boot = {
|
||||
binfmt.emulatedSystems = [ "x86_64-linux" ];
|
||||
loader = {
|
||||
systemd-boot.enable = true;
|
||||
efi.canTouchEfiVariables = true;
|
||||
};
|
||||
};
|
||||
services.openssh.enable = true;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue