From 30c6879b423e2efdd451e99783cdfe5551e0bdcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 19 Jul 2020 17:09:27 +0100 Subject: [PATCH 1/2] add validation mode --- pkgs/sops-install-secrets/main.go | 177 +++++++++++++++++++++---- pkgs/sops-install-secrets/main_test.go | 31 +++++ 2 files changed, 180 insertions(+), 28 deletions(-) diff --git a/pkgs/sops-install-secrets/main.go b/pkgs/sops-install-secrets/main.go index 6f09e3f..ddae4da 100644 --- a/pkgs/sops-install-secrets/main.go +++ b/pkgs/sops-install-secrets/main.go @@ -21,16 +21,16 @@ import ( ) type secret struct { - Name string `json:"name"` - Key string `json:"key"` - Path string `json:"path"` - Owner string `json:"owner"` - Group string `json:"group"` - SopsFile string `json:"sopsFile"` - Format string `json:"format"` - Mode string `json:"mode"` - RestartServices []string `json:"restartServices"` - ReloadServices []string `json:"reloadServices"` + Name string `json:"name"` + Key string `json:"key"` + Path string `json:"path"` + Owner string `json:"owner"` + Group string `json:"group"` + SopsFile string `json:"sopsFile"` + Format FormatType `json:"format"` + Mode string `json:"mode"` + RestartServices []string `json:"restartServices"` + ReloadServices []string `json:"reloadServices"` value []byte mode os.FileMode owner int @@ -45,6 +45,60 @@ type manifest struct { GnupgHome string `json:"gnupgHome"` } +type secretFile struct { + cipherText []byte + keys map[string]interface{} + /// First secret that defined this secretFile, used for error messages + firstSecret *secret +} + +type FormatType string + +const ( + Yaml FormatType = "yaml" + Json FormatType = "json" + Binary FormatType = "binary" +) + +func (f *FormatType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + var t = FormatType(s) + switch t { + case "": + *f = Yaml + case Yaml, Json, Binary: + *f = t + } + + return nil +} + +func (f FormatType) MarshalJSON() ([]byte, error) { + return json.Marshal(string(f)) +} + +type CheckMode string + +const ( + Manifest CheckMode = "manifest" + SopsFile CheckMode = "sopsfile" + Off CheckMode = "off" +) + +type options struct { + checkMode CheckMode + manifest string +} + +type appContext struct { + manifest manifest + secretFiles map[string]secretFile + checkMode CheckMode +} + func readManifest(path string) (*manifest, error) { file, err := os.Open(path) if err != nil { @@ -103,14 +157,14 @@ type plainData struct { func decryptSecret(s *secret, sourceFiles map[string]plainData) error { sourceFile := sourceFiles[s.SopsFile] if sourceFile.data == nil || sourceFile.binary == nil { - plain, err := decrypt.File(s.SopsFile, s.Format) + plain, err := decrypt.File(s.SopsFile, string(s.Format)) if err != nil { return fmt.Errorf("Failed to decrypt '%s': %s", s.SopsFile, err) } - if s.Format == "binary" { + if s.Format == Binary { sourceFile.binary = plain } else { - if s.Format == "yaml" { + if s.Format == Yaml { if err := yaml.Unmarshal(plain, &sourceFile.data); err != nil { return fmt.Errorf("Cannot parse yaml of '%s': %s", s.SopsFile, err) } @@ -121,7 +175,7 @@ func decryptSecret(s *secret, sourceFiles map[string]plainData) error { } } } - if s.Format == "binary" { + if s.Format == Binary { s.value = sourceFile.binary } else { val, ok := sourceFile.data[s.Key] @@ -215,7 +269,55 @@ func lookupKeysGroup() (int, error) { return int(gid), nil } -func validateSecret(secret *secret) error { +func (app *appContext) loadSopsFile(s *secret) (*secretFile, error) { + if app.checkMode == Manifest { + return &secretFile{firstSecret: s}, nil + } + + cipherText, err := ioutil.ReadFile(s.SopsFile) + if err != nil { + return nil, fmt.Errorf("Failed reading %s: %s", s.SopsFile, err) + } + + var keys map[string]interface{} + if s.Format == Binary { + if err := json.Unmarshal(cipherText, &keys); err != nil { + return nil, fmt.Errorf("Cannot parse json of '%s': %s", s.SopsFile, err) + } + return &secretFile{cipherText: cipherText, firstSecret: s}, nil + } + + if s.Format == Yaml { + if err := yaml.Unmarshal(cipherText, &keys); err != nil { + return nil, fmt.Errorf("Cannot parse yaml of '%s': %s", s.SopsFile, err) + } + } else if err := json.Unmarshal(cipherText, &keys); err != nil { + return nil, fmt.Errorf("Cannot parse json of '%s': %s", s.SopsFile, err) + } + + return &secretFile{ + cipherText: cipherText, + keys: keys, + firstSecret: s, + }, nil + +} + +func (app *appContext) validateSopsFile(s *secret, file *secretFile) error { + if file.firstSecret.Format != s.Format { + return fmt.Errorf("secret %s defined the format of %s as %s, but it was specified as %s in %s before", + s.Name, s.SopsFile, s.Format, + file.firstSecret.Format, file.firstSecret.Name) + } + if app.checkMode != Manifest && s.Format != Binary { + if _, ok := file.keys[s.Key]; !ok { + return fmt.Errorf("secret %s with the key %s not found in %s", s.Name, s.Key, s.SopsFile) + } + } + return nil +} + +func (app *appContext) validateSecret(secret *secret) error { mode, err := strconv.ParseUint(secret.Mode, 8, 16) if err != nil { return fmt.Errorf("Invalid number in mode: %d: %s", mode, err) @@ -247,14 +349,24 @@ func validateSecret(secret *secret) error { } if secret.Format != "yaml" && secret.Format != "json" && secret.Format != "binary" { - return fmt.Errorf("Unsupported format %s for secret %s", - secret.Format, secret.Name) + return fmt.Errorf("Unsupported format %s for secret %s", secret.Format, secret.Name) } - return nil + file, ok := app.secretFiles[secret.SopsFile] + if !ok { + maybeFile, err := app.loadSopsFile(secret) + if err != nil { + return err + } + app.secretFiles[secret.SopsFile] = *maybeFile + file = *maybeFile + } + + return app.validateSopsFile(secret, &file) } -func validateManifest(m *manifest) error { +func (app *appContext) validateManifest() error { + m := &app.manifest if m.SecretsMountPoint == "" { m.SecretsMountPoint = "/run/secrets.d" } @@ -266,7 +378,7 @@ func validateManifest(m *manifest) error { "Both options are mutual exclusive.") } for i := range m.Secrets { - if err := validateSecret(&m.Secrets[i]); err != nil { + if err := app.validateSecret(&m.Secrets[i]); err != nil { return err } } @@ -358,11 +470,6 @@ func setupGPGKeyring(sshKeys []string, parentDir string) (*keyring, error) { return &k, nil } -type options struct { - check bool - manifest string -} - func parseFlags(args []string) (*options, error) { var opts options fs := flag.NewFlagSet(args[0], flag.ContinueOnError) @@ -370,11 +477,19 @@ func parseFlags(args []string) (*options, error) { fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [OPTION] manifest.json\n", args[0]) fs.PrintDefaults() } - fs.BoolVar(&opts.check, "check", false, "Validate manifest instead installing it") + var checkMode string + fs.StringVar(&checkMode, "check-mode", "off", `Validate configuration without installing it (possible values: "manifest","sopsfile","off")`) if err := fs.Parse(args[1:]); err != nil { return nil, err } + switch CheckMode(checkMode) { + case Manifest, SopsFile, Off: + opts.checkMode = CheckMode(checkMode) + default: + return nil, fmt.Errorf("Invalid value provided for -check-mode flag: %s", opts.checkMode) + } + if fs.NArg() != 1 { flag.Usage() return nil, flag.ErrHelp @@ -394,11 +509,17 @@ func installSecrets(args []string) error { return err } - if err := validateManifest(manifest); err != nil { + app := appContext{ + manifest: *manifest, + checkMode: opts.checkMode, + secretFiles: make(map[string]secretFile), + } + + if err := app.validateManifest(); err != nil { return fmt.Errorf("Manifest is not valid: %s", err) } - if opts.check { + if app.checkMode != Off { return nil } diff --git a/pkgs/sops-install-secrets/main_test.go b/pkgs/sops-install-secrets/main_test.go index fedee8a..895b3e4 100644 --- a/pkgs/sops-install-secrets/main_test.go +++ b/pkgs/sops-install-secrets/main_test.go @@ -213,3 +213,34 @@ func TestAll(t *testing.T) { testGPG(t) testSSHKey(t) } + +func TestValidateManifest(t *testing.T) { + assets := testAssetPath() + + testdir := newTestDir(t) + defer testdir.Remove() + + s := secret{ + Name: "test", + Key: "test_key", + Owner: "nobody", + Group: "nogroup", + SopsFile: path.Join(assets, "secrets.yaml"), + Path: path.Join(testdir.path, "test-target"), + Mode: "0400", + RestartServices: []string{}, + ReloadServices: make([]string, 0), + } + + m := manifest{ + Secrets: []secret{s}, + SecretsMountPoint: testdir.secretsPath, + SymlinkPath: testdir.symlinkPath, + SSHKeyPaths: []string{"non-existing-key"}, + } + + path := writeManifest(t, testdir.path, &m) + + ok(t, installSecrets([]string{"sops-install-secrets", "-check-mode=manifest", path})) + ok(t, installSecrets([]string{"sops-install-secrets", "-check-mode=sopsfile", path})) +} From b395301115ec01621ecf80b7c0142431381434bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 19 Jul 2020 19:13:48 +0100 Subject: [PATCH 2/2] nixos: enable manifest validation --- README.md | 3 ++- modules/sops/default.nix | 30 +++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 72c2f1f..d10a25a 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,12 @@ key management APIs such as AWS KMS, GCP KMS, Azure Key Vault or Hashicorp's vau - Compatible with all NixOS deployment frameworks: [NixOps](https://github.com/NixOS/nixops), nixos-rebuild, [krops](https://github.com/krebs/krops/), [morph](https://github.com/DBCDK/morph), [nixus](https://github.com/Infinisil/nixus) - Version-control friendly: Since all files are encrypted they can directly committed to version control. The format is readable in diffs and there are also ways of showing [git diffs in cleartext](https://github.com/mozilla/sops#showing-diffs-in-cleartext-in-git) -- CI friendly: Since nixops files can be added to the nix store as well without leaking secrets, machine definition can be build as a whole. +- CI friendly: Since sops files can be added to the nix store as well without leaking secrets, machine definition can be build as a whole. - Atomic upgrades: New secrets are written to a new directory which replaces the old directory in an atomic step. - Rollback support: If sops files are added to Nix store, old secrets can be rolled back. This is optional. - Fast: Unlike solutions implemented by NixOps, krops and morph there is no extra step required to upload secrets - Different storage formats: Secrets can be stored in Yaml, JSON or binary. +- Minimize configuration errors: sops files are checked against the configuration at evluation time. ## Usage example diff --git a/modules/sops/default.nix b/modules/sops/default.nix index f972866..b339f19 100644 --- a/modules/sops/default.nix +++ b/modules/sops/default.nix @@ -80,6 +80,13 @@ let symlinkPath = "/run/secrets"; inherit (cfg) gnupgHome sshKeyPaths; }); + + checkedManifest = pkgs.runCommandNoCC "checked-manifest.json" { + nativeBuildInputs = [ sops-install-secrets ]; + } '' + sops-install-secrets -check-mode=${if cfg.validateSopsFiles then "sopsfile" else "manifest"} ${manifest} + cp ${manifest} $out + ''; in { options.sops = { secrets = mkOption { @@ -97,6 +104,15 @@ in { ''; }; + validateSopsFiles = mkOption { + type = types.bool; + default = true; + description = '' + Check all sops files at evaluation time. + This requires sops files to be added to the nix store. + ''; + }; + gnupgHome = mkOption { type = types.nullOr types.str; default = null; @@ -118,18 +134,22 @@ in { }; }; config = mkIf (cfg.secrets != {}) { - assertions = [{ assertion = cfg.gnupgHome != null -> cfg.sshKeyPaths == []; - message = "config.sops.gnupgHome and config.sops.sshKeyPaths are mutual exclusive"; + message = "Configuration options sops.gnupgHome and sops.sshKeyPaths cannot be set both at the same time"; } { assertion = cfg.gnupgHome == null -> cfg.sshKeyPaths != []; - message = "Either config.sops.sshKeyPaths and config.sops.gnupgHome must be set"; - }]; + message = "Either sops.sshKeyPaths and sops.gnupgHome must be set"; + }] ++ map (name: let + inherit (cfg.secrets.${name}) sopsFile; + in { + assertion = cfg.validateSopsFiles -> builtins.isPath sopsFile; + message = "${sopsFile} is not in the nix store. Either add it to the nix store or set `sops.validateSopsFiles` to false"; + }) (builtins.attrNames cfg.secrets); system.activationScripts.setup-secrets = stringAfter [ "users" "groups" ] '' echo setting up secrets... - ${optionalString (cfg.gnupgHome != null) "SOPS_GPG_EXEC=${pkgs.gnupg}/bin/gpg"} ${sops-install-secrets}/bin/sops-install-secrets ${manifest} + ${optionalString (cfg.gnupgHome != null) "SOPS_GPG_EXEC=${pkgs.gnupg}/bin/gpg"} ${sops-install-secrets}/bin/sops-install-secrets ${checkedManifest} ''; }; }