diff --git a/go.mod b/go.mod index c292abf67..3e801ea77 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/fatih/color v1.19.0 github.com/getsops/gopgagent v0.0.0-20241224165529-7044f28e491e + github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/go-cmp v0.7.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/goware/prefixer v0.0.0-20160118172347-395022866408 @@ -100,7 +101,6 @@ require ( github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-yaml v1.9.8 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/google/s2a-go v0.1.9 // indirect diff --git a/go.sum b/go.sum index 5c83cabe2..01a2c353e 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-yaml v1.9.8 h1:5gMyLUeU1/6zl+WFfR1hN7D2kf+1/eRGa7DFtToiBvQ= github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index 991b582c9..1cbe48cae 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -3,7 +3,6 @@ package dotenv //import "github.com/getsops/sops/v3/stores/dotenv" import ( "bytes" "fmt" - "sort" "strings" "github.com/getsops/sops/v3" @@ -33,47 +32,15 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { if err != nil { return sops.Tree{}, err } - - var resultBranch sops.TreeBranch - mdMap := make(map[string]interface{}) - for _, item := range branches[0] { - switch key := item.Key.(type) { - case string: - if strings.HasPrefix(key, SopsPrefix) { - key = key[len(SopsPrefix):] - mdMap[key] = item.Value - } else { - resultBranch = append(resultBranch, item) - } - case sops.Comment: - resultBranch = append(resultBranch, item) - default: - panic(fmt.Sprintf("Unexpected type: %T (value %#v)", key, key)) - } - } - if len(mdMap) == 0 { - return sops.Tree{}, sops.MetadataNotFound - } - - stores.DecodeNewLines(mdMap) - err = stores.DecodeNonStrings(mdMap) - if err != nil { - return sops.Tree{}, err - } - metadata, err := stores.UnflattenMetadata(mdMap) + branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenFull, + }) if err != nil { return sops.Tree{}, err } - internalMetadata, err := metadata.ToInternal() - if err != nil { - return sops.Tree{}, err - } - return sops.Tree{ - Branches: sops.TreeBranches{ - resultBranch, - }, - Metadata: internalMetadata, + Branches: branches, + Metadata: metadata, }, nil } @@ -111,29 +78,13 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { // EmitEncryptedFile returns the encrypted file's bytes corresponding to a sops // runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { - metadata := stores.MetadataFromInternal(in.Metadata) - mdItems, err := stores.FlattenMetadata(metadata) + branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenFull, + }) if err != nil { - return nil, err - } - - stores.EncodeNonStrings(mdItems) - stores.EncodeNewLines(mdItems) - - var keys []string - for k := range mdItems { - keys = append(keys, k) - } - sort.Strings(keys) - - for _, key := range keys { - var value = mdItems[key] - if value == nil { - continue - } - in.Branches[0] = append(in.Branches[0], sops.TreeItem{Key: SopsPrefix + key, Value: value}) + return nil, fmt.Errorf("Error marshaling metadata: %s", err) } - return store.EmitPlainFile(in.Branches) + return store.EmitPlainFile(branches) } // EmitPlainFile returns the plaintext file's bytes corresponding to a sops diff --git a/stores/flatten.go b/stores/flatten.go index f961d74d0..4ba0a3e06 100644 --- a/stores/flatten.go +++ b/stores/flatten.go @@ -1,81 +1,20 @@ package stores import ( - "encoding/json" "fmt" + "math" + "sort" "strconv" "strings" + + "github.com/getsops/sops/v3" ) const mapSeparator = "__map_" const listSeparator = "__list_" -// flattenAndMerge flattens the provided value and merges into the -// into map using prefix -func flattenAndMerge(into map[string]interface{}, prefix string, value interface{}) { - flattenedValue := flattenValue(value) - if flattenedValue, ok := flattenedValue.(map[string]interface{}); ok { - for flatK, flatV := range flattenedValue { - into[prefix+flatK] = flatV - } - } else { - into[prefix] = value - } -} - -func flattenValue(value interface{}) interface{} { - var output interface{} - switch value := value.(type) { - case map[string]interface{}: - newMap := make(map[string]interface{}) - for k, v := range value { - flattenAndMerge(newMap, mapSeparator+k, v) - } - output = newMap - case []interface{}: - newMap := make(map[string]interface{}) - for i, v := range value { - flattenAndMerge(newMap, listSeparator+fmt.Sprintf("%d", i), v) - } - output = newMap - default: - output = value - } - return output -} - -// Flatten flattens a map with potentially nested maps into a flat -// map. Only string keys are allowed on both the top-level map and -// child maps. -func Flatten(in map[string]interface{}) map[string]interface{} { - newMap := make(map[string]interface{}) - for k, v := range in { - if flat, ok := flattenValue(v).(map[string]interface{}); ok { - for flatK, flatV := range flat { - newMap[k+flatK] = flatV - } - } else { - newMap[k] = v - } - } - return newMap -} - -// FlattenMetadata flattens a Metadata struct into a flat map. -func FlattenMetadata(md Metadata) (map[string]interface{}, error) { - var mdMap map[string]interface{} - jsonBytes, err := json.Marshal(md) - if err != nil { - return nil, err - } - err = json.Unmarshal(jsonBytes, &mdMap) - if err != nil { - return nil, err - } - - flat := Flatten(mdMap) - return flat, nil -} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Unflatten type token interface{} @@ -130,162 +69,189 @@ func tokenize(path string) []token { return tokens } -// unflatten takes the currentNode, currentToken, nextToken and value -// and populates currentNode such that currentToken can be considered -// processed. It inspects nextToken to decide what type to allocate -// and assign under currentNode. -func unflatten(currentNode interface{}, currentToken, nextToken token, value interface{}) interface{} { - switch currentToken := currentToken.(type) { - case mapToken: - currentNode := currentNode.(map[string]interface{}) - switch nextToken := nextToken.(type) { +type node struct { + value *interface{} + subkeys map[string]*node + indices map[int]*node +} + +func place(value *interface{}, path []token, root *node) error { + for _, token := range path { + var next *node + var ok bool + switch t := token.(type) { case mapToken: - if _, ok := currentNode[currentToken.key]; !ok { - currentNode[currentToken.key] = make(map[string]interface{}) + if root.subkeys == nil { + root.subkeys = make(map[string]*node) } - next := currentNode[currentToken.key].(map[string]interface{}) - return next - case listToken: - if _, ok := currentNode[currentToken.key]; !ok { - currentNode[currentToken.key] = make([]interface{}, nextToken.position+1) - } - next := currentNode[currentToken.key].([]interface{}) - if nextToken.position >= len(next) { - // Grow the slice and reassign it - newNext := make([]interface{}, nextToken.position+1) - copy(newNext, next) - next = newNext - currentNode[currentToken.key] = next - } - return next - default: - currentNode[currentToken.key] = value - } - case listToken: - currentNode := currentNode.([]interface{}) - switch nextToken := nextToken.(type) { - case mapToken: - if currentNode[currentToken.position] == nil { - currentNode[currentToken.position] = make(map[string]interface{}) + next, ok = root.subkeys[t.key] + if !ok { + next = &node{} + root.subkeys[t.key] = next } - next := currentNode[currentToken.position].(map[string]interface{}) - return next case listToken: - if currentNode[currentToken.position] == nil { - currentNode[currentToken.position] = make([]interface{}, nextToken.position+1) + if root.indices == nil { + root.indices = make(map[int]*node) } - next := currentNode[currentToken.position].([]interface{}) - if nextToken.position >= len(next) { - // Grow the slice and reassign it - newNext := make([]interface{}, nextToken.position+1) - copy(newNext, next) - next = newNext - currentNode[currentToken.position] = next + next, ok = root.indices[t.position] + if !ok { + next = &node{} + root.indices[t.position] = next } - return next - default: - currentNode[currentToken.position] = value } + root = next + } + if root.value != nil { + return fmt.Errorf("Duplicate value") } + root.value = value return nil } -// Unflatten unflattens a map flattened by Flatten -func Unflatten(in map[string]interface{}) map[string]interface{} { - newMap := make(map[string]interface{}) - for k, v := range in { - var current interface{} = newMap - tokens := append(tokenize(k), nil) - for i := 0; i < len(tokens)-1; i++ { - current = unflatten(current, tokens[i], tokens[i+1], v) - } +func b2i(value bool) int { + if value { + return 1 } - return newMap + return 0 } -// UnflattenMetadata unflattens a map flattened by FlattenMetadata into Metadata -func UnflattenMetadata(in map[string]interface{}) (Metadata, error) { - m := Unflatten(in) - var md Metadata - jsonBytes, err := json.Marshal(m) - if err != nil { - return md, err +func convert(root *node) (interface{}, error) { + hasValue := root.value != nil + hasSubkey := len(root.subkeys) > 0 + hasIndex := len(root.indices) > 0 + if b2i(hasValue)+b2i(hasSubkey)+b2i(hasIndex) > 1 { + return nil, fmt.Errorf("Type mismatch") } - err = json.Unmarshal(jsonBytes, &md) - return md, err -} - -// DecodeNewLines replaces \\n with \n for all string values in the map. -// Used by config stores that do not handle multi-line values (ini, dotenv). -func DecodeNewLines(m map[string]interface{}) { - for k, v := range m { - if s, ok := v.(string); ok { - m[k] = strings.Replace(s, "\\n", "\n", -1) + if hasValue { + return *root.value, nil + } + if hasSubkey { + keys := make([]string, len(root.subkeys)) + index := 0 + for k := range root.subkeys { + keys[index] = k + index += 1 } + sort.Strings(keys) + result := make(sops.TreeBranch, len(keys)) + for index, key := range keys { + value, err := convert(root.subkeys[key]) + if err != nil { + return nil, err + } + result[index] = sops.TreeItem{ + Key: key, + Value: value, + } + } + return result, nil } -} - -// EncodeNewLines replaces \n with \\n for all string values in the map. -// Used by config stores that do not handle multi-line values (ini, dotenv). -func EncodeNewLines(m map[string]interface{}) { - for k, v := range m { - if s, ok := v.(string); ok { - m[k] = strings.Replace(s, "\n", "\\n", -1) + minValue := math.MaxInt + maxValue := math.MinInt + for k := range root.indices { + if k < minValue { + minValue = k } + if k > maxValue { + maxValue = k + } + } + if minValue != 0 || maxValue+1 != len(root.indices) { + return nil, fmt.Errorf("Incomplete list") + } + result := make([]interface{}, maxValue+1) + for k, v := range root.indices { + value, err := convert(v) + if err != nil { + return nil, err + } + result[k] = value } + return result, nil } -// DecodeNonStrings will look for known metadata keys that are not strings and decode to the appropriate type -func DecodeNonStrings(m map[string]interface{}) error { - if v, ok := m["mac_only_encrypted"]; ok { - m["mac_only_encrypted"] = false - if v == "true" { - m["mac_only_encrypted"] = true +func unflattenTreeBranch(branch sops.TreeBranch) (sops.TreeBranch, error) { + root := &node{} + for _, item := range branch { + if _, ok := item.Key.(sops.Comment); ok { + continue } - } - if v, ok := m["shamir_threshold"]; ok { - switch val := v.(type) { - case string: - vInt, err := strconv.Atoi(val) + if key, ok := item.Key.(string); ok { + tokens := tokenize(key) + err := place(&item.Value, tokens, root) if err != nil { - // Older versions of SOPS stored shamir_threshold as a floating point representation - // of the actual integer. Try to parse a floating point number and see whether it - // can be converted without loss to an integer. - vFloat, floatErr := strconv.ParseFloat(val, 64) - vInt = int(vFloat) - if floatErr != nil || float64(vInt) != vFloat { - return fmt.Errorf("shamir_threshold is not an integer: %s", err.Error()) - } + return nil, fmt.Errorf("Error while unflattening %q: %w", key, err) } - m["shamir_threshold"] = vInt - case int: - m["shamir_threshold"] = val - default: - return fmt.Errorf("shamir_threshold is neither a string nor an integer, but %T", val) + } else { + return nil, fmt.Errorf("Found non-string key %q when unflattening", item.Key) } } - return nil + result, err := convert(root) + if err != nil { + return nil, fmt.Errorf("Error while unflattening: %w", err) + } + if tb, ok := result.(sops.TreeBranch); ok { + return tb, nil + } + return nil, fmt.Errorf("Internal error: cannot find root") +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Flatten + +func flattenDescendValue(value interface{}, key string, destination sops.TreeBranch, destinationMap *map[string]bool) (sops.TreeBranch, error) { + switch value := value.(type) { + case sops.TreeBranch: + return flattenDescendMap(value, key+mapSeparator, destination, destinationMap) + case []interface{}: + return flattenDescendArray(value, key+listSeparator, destination, destinationMap) + } + if _, ok := (*destinationMap)[key]; ok { + return nil, fmt.Errorf("Found key collision %q while flattening", key) + } + destination = append(destination, sops.TreeItem{ + Key: key, + Value: value, + }) + (*destinationMap)[key] = true + return destination, nil } -// EncodeNonStrings will look for known metadata keys that are not strings and will encode it to strings -func EncodeNonStrings(m map[string]interface{}) { - if v, found := m["mac_only_encrypted"]; found { - if vBool, ok := v.(bool); ok { - m["mac_only_encrypted"] = "false" - if vBool { - m["mac_only_encrypted"] = "true" +func flattenDescendMap(branch sops.TreeBranch, prefix string, destination sops.TreeBranch, destinationMap *map[string]bool) (sops.TreeBranch, error) { + for _, item := range branch { + if _, ok := item.Key.(sops.Comment); ok { + continue + } + if key, ok := item.Key.(string); ok { + var err error + destination, err = flattenDescendValue(item.Value, prefix+key, destination, destinationMap) + if err != nil { + return nil, err } + } else { + return nil, fmt.Errorf("Found non-string key %q when flattening", item.Key) } } - if v, found := m["shamir_threshold"]; found { - if vInt, ok := v.(int); ok { - m["shamir_threshold"] = fmt.Sprintf("%d", vInt) + return destination, nil +} + +func flattenDescendArray(array []interface{}, prefix string, destination sops.TreeBranch, destinationMap *map[string]bool) (sops.TreeBranch, error) { + i := 0 + for _, item := range array { + if _, ok := item.(sops.Comment); ok { + continue } - // FlattenMetadata serializes the input as JSON and then deserializes it. - // The JSON unserializer treats every number as a float, so the above 'if' - // never applies in that situation. - if vFloat, ok := v.(float64); ok { - m["shamir_threshold"] = fmt.Sprintf("%.0f", vFloat) + var err error + destination, err = flattenDescendValue(item, fmt.Sprintf("%s%d", prefix, i), destination, destinationMap) + if err != nil { + return nil, err } + i++ } + return destination, nil +} + +func flattenTreeBranch(branch sops.TreeBranch, prefix string) (sops.TreeBranch, error) { + destinationMap := map[string]bool{} + return flattenDescendMap(branch, prefix, nil, &destinationMap) } diff --git a/stores/flatten_test.go b/stores/flatten_test.go index 6c75da3d1..e2552a110 100644 --- a/stores/flatten_test.go +++ b/stores/flatten_test.go @@ -3,115 +3,10 @@ package stores import ( "testing" + "github.com/getsops/sops/v3" "github.com/stretchr/testify/assert" ) -func TestFlat(t *testing.T) { - input := map[string]interface{}{ - "foo": "bar", - } - expected := map[string]interface{}{ - "foo": "bar", - } - flattened := Flatten(input) - assert.Equal(t, expected, flattened) - unflattened := Unflatten(flattened) - assert.Equal(t, input, unflattened) -} - -func TestMap(t *testing.T) { - input := map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": 0, - "baz": 0, - }, - } - expected := map[string]interface{}{ - "foo" + mapSeparator + "bar": 0, - "foo" + mapSeparator + "baz": 0, - } - flattened := Flatten(input) - assert.Equal(t, expected, flattened) - unflattened := Unflatten(flattened) - assert.Equal(t, input, unflattened) -} - -func TestFlattenMapMoreNesting(t *testing.T) { - input := map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": map[string]interface{}{ - "baz": 0, - }, - }, - } - expected := map[string]interface{}{ - "foo" + mapSeparator + "bar" + mapSeparator + "baz": 0, - } - flattened := Flatten(input) - assert.Equal(t, expected, flattened) - unflattened := Unflatten(flattened) - assert.Equal(t, input, unflattened) -} - -func TestFlattenList(t *testing.T) { - input := map[string]interface{}{ - "foo": []interface{}{ - 0, - }, - } - expected := map[string]interface{}{ - "foo" + listSeparator + "0": 0, - } - flattened := Flatten(input) - assert.Equal(t, expected, flattened) - unflattened := Unflatten(flattened) - assert.Equal(t, input, unflattened) -} - -func TestFlattenListWithMap(t *testing.T) { - input := map[string]interface{}{ - "foo": []interface{}{ - map[string]interface{}{ - "bar": 0, - }, - }, - } - expected := map[string]interface{}{ - "foo" + listSeparator + "0" + mapSeparator + "bar": 0, - } - flattened := Flatten(input) - assert.Equal(t, expected, flattened) - unflattened := Unflatten(flattened) - assert.Equal(t, input, unflattened) -} - -func TestFlatten(t *testing.T) { - input := map[string]interface{}{ - "foo": "bar", - "baz": map[string]interface{}{ - "foo": 2, - "bar": map[string]interface{}{ - "foo": 2, - }, - }, - "qux": []interface{}{ - "hello", 1, 2, - }, - } - expected := map[string]interface{}{ - "foo": "bar", - "baz" + mapSeparator + "foo": 2, - "baz" + mapSeparator + "bar" + mapSeparator + "foo": 2, - "qux" + listSeparator + "0": "hello", - "qux" + listSeparator + "1": 1, - "qux" + listSeparator + "2": 2, - } - flattened := Flatten(input) - assert.Equal(t, expected, flattened) - unflattened := Unflatten(flattened) - assert.Equal(t, input, unflattened) -} - func TestTokenizeFlat(t *testing.T) { input := "bar" expected := []token{mapToken{"bar"}} @@ -140,130 +35,280 @@ func TestTokenizeNested(t *testing.T) { assert.Equal(t, expected, tokenized) } -func TestFlattenMetadata(t *testing.T) { - tests := []struct { - input Metadata - want map[string]interface{} - }{ - {Metadata{MACOnlyEncrypted: false}, map[string]interface{}{"mac_only_encrypted": nil}}, - {Metadata{MACOnlyEncrypted: true}, map[string]interface{}{"mac_only_encrypted": true}}, - {Metadata{MessageAuthenticationCode: "line1\nline2"}, map[string]interface{}{"mac": "line1\nline2"}}, - {Metadata{MessageAuthenticationCode: "line1\n\n\nline2\n\nline3"}, map[string]interface{}{"mac": "line1\n\n\nline2\n\nline3"}}, - } +func TestUnflattenTreeBranch(t *testing.T) { + var ( + input = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "key2__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "key2__list_1__map_foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "key2__list_1__map_baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "key3__map_foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "key3__map_baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "key4__map_foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "key4__map_baz", + Value: "bam", + }, + } + expectedOutput = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1", + Value: []interface{}{ + "foo", + }, + }, + sops.TreeItem{ + Key: "key2", + Value: []interface{}{ + "foo", + sops.TreeBranch{ + sops.TreeItem{ + Key: "baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + }, + sops.TreeItem{ + Key: "key3", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + sops.TreeItem{ + Key: "key4", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + } - for _, tt := range tests { - got, err := FlattenMetadata(tt.input) - assert.NoError(t, err) - for k, v := range tt.want { - assert.Equal(t, v, got[k]) + inputDupe = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "key1__list_0", + Value: "bar", + }, } - } -} -func TestFlattenMetadataToUnflattenMetadata(t *testing.T) { - tests := []struct { - input Metadata - }{ - {Metadata{MACOnlyEncrypted: true}}, - {Metadata{MACOnlyEncrypted: false}}, - {Metadata{ShamirThreshold: 3}}, - {Metadata{MessageAuthenticationCode: "line1\nline2"}}, - {Metadata{MessageAuthenticationCode: "line1\n\n\nline2\n\nline3"}}, - } + inputSkip1 = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "key1__list_999999999999", + Value: "bar", + }, + } - for _, tt := range tests { - flat, err := FlattenMetadata(tt.input) - assert.NoError(t, err) - md, err := UnflattenMetadata(flat) - assert.NoError(t, err) - assert.Equal(t, tt.input, md) - } -} + inputSkip2 = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1__list_1", + Value: "foo", + }, + } -func TestDecodeNewLines(t *testing.T) { - tests := []struct { - input map[string]interface{} - want map[string]interface{} - }{ - {map[string]interface{}{"mac": "line1\\nline2"}, map[string]interface{}{"mac": "line1\nline2"}}, - {map[string]interface{}{"mac": "line1\\n\\n\\nline2\\n\\nline3"}, map[string]interface{}{"mac": "line1\n\n\nline2\n\nline3"}}, - } + inputCollision1 = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "key1__map_foo", + Value: "bar", + }, + } - for _, tt := range tests { - DecodeNewLines(tt.input) - for k, v := range tt.want { - assert.Equal(t, v, tt.input[k]) + inputCollision2 = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1", + Value: "foo", + }, + sops.TreeItem{ + Key: "key1__list_0", + Value: "bar", + }, } - } -} + ) -func TestEncodeNewLines(t *testing.T) { - tests := []struct { - input map[string]interface{} - want map[string]interface{} - }{ - {map[string]interface{}{"mac": "line1\nline2"}, map[string]interface{}{"mac": "line1\\nline2"}}, - {map[string]interface{}{"mac": "line1\n\n\nline2\n\nline3"}, map[string]interface{}{"mac": "line1\\n\\n\\nline2\\n\\nline3"}}, - } + output, err := unflattenTreeBranch(input) + assert.Nil(t, err) + assert.Equal(t, output, expectedOutput) - for _, tt := range tests { - EncodeNewLines(tt.input) - for k, v := range tt.want { - assert.Equal(t, v, tt.input[k]) - } - } -} + output, err = unflattenTreeBranch(inputDupe) + assert.NotNil(t, err) + assert.Equal(t, "Error while unflattening \"key1__list_0\": Duplicate value", err.Error()) + assert.Nil(t, output) -func TestDecodeNonStrings(t *testing.T) { - tests := []struct { - input map[string]interface{} - want map[string]interface{} - }{ - {map[string]interface{}{"mac_only_encrypted": "false"}, map[string]interface{}{"mac_only_encrypted": false}}, - {map[string]interface{}{"mac_only_encrypted": "true"}, map[string]interface{}{"mac_only_encrypted": true}}, - {map[string]interface{}{"mac_only_encrypted": "something-else"}, map[string]interface{}{"mac_only_encrypted": false}}, - {map[string]interface{}{"shamir_threshold": "2"}, map[string]interface{}{"shamir_threshold": 2}}, - {map[string]interface{}{"shamir_threshold": "002"}, map[string]interface{}{"shamir_threshold": 2}}, - {map[string]interface{}{"shamir_threshold": "123"}, map[string]interface{}{"shamir_threshold": 123}}, - {map[string]interface{}{"shamir_threshold": 123}, map[string]interface{}{"shamir_threshold": 123}}, - } + output, err = unflattenTreeBranch(inputSkip1) + assert.NotNil(t, err) + assert.Equal(t, "Error while unflattening: Incomplete list", err.Error()) + assert.Nil(t, output) - for _, tt := range tests { - err := DecodeNonStrings(tt.input) - assert.Nil(t, err) - assert.Equal(t, tt.want, tt.input) - } -} + output, err = unflattenTreeBranch(inputSkip2) + assert.NotNil(t, err) + assert.Equal(t, "Error while unflattening: Incomplete list", err.Error()) + assert.Nil(t, output) -func TestDecodeNonStringsErrors(t *testing.T) { - tests := []struct { - input map[string]interface{} - want string - }{ - {map[string]interface{}{"shamir_threshold": "foo"}, "shamir_threshold is not an integer: strconv.Atoi: parsing \"foo\": invalid syntax"}, - {map[string]interface{}{"shamir_threshold": true}, "shamir_threshold is neither a string nor an integer, but bool"}, - } + output, err = unflattenTreeBranch(inputCollision1) + assert.NotNil(t, err) + assert.Equal(t, "Error while unflattening: Type mismatch", err.Error()) + assert.Nil(t, output) - for _, tt := range tests { - err := DecodeNonStrings(tt.input) - assert.NotNil(t, err) - assert.Equal(t, tt.want, err.Error()) - } + output, err = unflattenTreeBranch(inputCollision2) + assert.NotNil(t, err) + assert.Equal(t, "Error while unflattening: Type mismatch", err.Error()) + assert.Nil(t, output) } -func TestEncodeNonStrings(t *testing.T) { - tests := []struct { - input map[string]interface{} - want map[string]interface{} - }{ - {map[string]interface{}{"mac_only_encrypted": false}, map[string]interface{}{"mac_only_encrypted": "false"}}, - {map[string]interface{}{"mac_only_encrypted": true}, map[string]interface{}{"mac_only_encrypted": "true"}}, - {map[string]interface{}{"shamir_threshold": 2}, map[string]interface{}{"shamir_threshold": "2"}}, - {map[string]interface{}{"shamir_threshold": 123}, map[string]interface{}{"shamir_threshold": "123"}}, - } +func TestFlattenTreeBranch(t *testing.T) { + var ( + input = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1", + Value: []interface{}{ + "foo", + }, + }, + sops.TreeItem{ + Key: "key2", + Value: []interface{}{ + "foo", + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "baz", + Value: "bam", + }, + }, + }, + }, + sops.TreeItem{ + Key: "key3", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "baz", + Value: "bam", + }, + }, + }, + sops.TreeItem{ + Key: "key4", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "baz", + Value: "bam", + }, + }, + }, + } + expectedOutput = sops.TreeBranch{ + sops.TreeItem{ + Key: "prefixkey1__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "prefixkey2__list_0", + Value: "foo", + }, + sops.TreeItem{ + Key: "prefixkey2__list_1__map_foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "prefixkey2__list_1__map_baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "prefixkey3__map_foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "prefixkey3__map_baz", + Value: "bam", + }, + sops.TreeItem{ + Key: "prefixkey4__map_foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "prefixkey4__map_baz", + Value: "bam", + }, + } + + inputDupe = sops.TreeBranch{ + sops.TreeItem{ + Key: "key1", + Value: "foo", + }, + sops.TreeItem{ + Key: "key1", + Value: "bar", + }, + } + ) + + output, err := flattenTreeBranch(input, "prefix") + assert.Nil(t, err) + assert.Equal(t, output, expectedOutput) - for _, tt := range tests { - EncodeNonStrings(tt.input) - assert.Equal(t, tt.want, tt.input) - } + output, err = flattenTreeBranch(inputDupe, "prefix") + assert.NotNil(t, err) + assert.Equal(t, "Found key collision \"prefixkey1\" while flattening", err.Error()) + assert.Nil(t, output) } diff --git a/stores/ini/store.go b/stores/ini/store.go index 79dd85eb3..8d49dc5ec 100644 --- a/stores/ini/store.go +++ b/stores/ini/store.go @@ -133,58 +133,22 @@ func (store Store) treeItemFromSection(section *ini.Section) (sops.TreeItem, err // LoadEncryptedFile loads encrypted INI file's bytes onto a sops.Tree runtime object func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { - iniFileOuter, err := ini.LoadSources(ini.LoadOptions{AllowNonUniqueSections: true}, in) + branches, err := store.LoadPlainFile(in) if err != nil { return sops.Tree{}, err } - - sopsSection, err := iniFileOuter.GetSection(stores.SopsMetadataKey) - if err != nil { - return sops.Tree{}, sops.MetadataNotFound - } - - metadataHolder, err := store.iniSectionToMetadata(sopsSection) + branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenBelowTop, + }) if err != nil { return sops.Tree{}, err } - - metadata, err := metadataHolder.ToInternal() - if err != nil { - return sops.Tree{}, err - } - // After that, we load the whole file into a map. - branches, err := store.treeBranchesFromIni(in) - if err != nil { - return sops.Tree{}, fmt.Errorf("Could not unmarshal input data: %s", err) - } - // Discard metadata, as we already loaded it. - for bi, branch := range branches { - for s, sectionBranch := range branch { - if sectionBranch.Key == stores.SopsMetadataKey { - branch = append(branch[:s], branch[s+1:]...) - branches[bi] = branch - } - } - } return sops.Tree{ Branches: branches, Metadata: metadata, }, nil } -func (store *Store) iniSectionToMetadata(sopsSection *ini.Section) (stores.Metadata, error) { - metadataHash := make(map[string]interface{}) - for k, v := range sopsSection.KeysHash() { - metadataHash[k] = v - } - stores.DecodeNewLines(metadataHash) - err := stores.DecodeNonStrings(metadataHash) - if err != nil { - return stores.Metadata{}, err - } - return stores.UnflattenMetadata(metadataHash) -} - // LoadPlainFile loads a plaintext INI file's bytes onto a sops.TreeBranches runtime object func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { branches, err := store.treeBranchesFromIni(in) @@ -197,47 +161,20 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { // EmitEncryptedFile returns encrypted INI file bytes corresponding to a sops.Tree // runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { - - metadata := stores.MetadataFromInternal(in.Metadata) - newBranch, err := store.encodeMetadataToIniBranch(metadata) + branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenBelowTop, + }) if err != nil { - return nil, err - } - sectionItem := sops.TreeItem{Key: stores.SopsMetadataKey, Value: newBranch} - branch := sops.TreeBranch{sectionItem} - - in.Branches = append(in.Branches, branch) - - out, err := store.iniFromTreeBranches(in.Branches) - if err != nil { - return nil, fmt.Errorf("Error marshaling to ini: %s", err) - } - return out, nil -} - -func (store *Store) encodeMetadataToIniBranch(md stores.Metadata) (sops.TreeBranch, error) { - flat, err := stores.FlattenMetadata(md) - if err != nil { - return nil, err - } - stores.EncodeNonStrings(flat) - stores.EncodeNewLines(flat) - - branch := sops.TreeBranch{} - for key, value := range flat { - if value == nil { - continue - } - branch = append(branch, sops.TreeItem{Key: key, Value: value}) + return nil, fmt.Errorf("Error marshaling metadata: %s", err) } - return branch, nil + return store.EmitPlainFile(branches) } // EmitPlainFile returns the plaintext INI file bytes corresponding to a sops.TreeBranches object func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { out, err := store.iniFromTreeBranches(in) if err != nil { - return nil, fmt.Errorf("Error marshaling to ini: %s", err) + return nil, fmt.Errorf("Error marshaling to INI: %s", err) } return out, nil } diff --git a/stores/json/store.go b/stores/json/store.go index 16b0c03ea..10ca7f2e5 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -280,46 +280,18 @@ func (store Store) reindentJSON(in []byte) ([]byte, error) { // LoadEncryptedFile loads an encrypted secrets file onto a sops.Tree object func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { - // Because we don't know what fields the input file will have, we have to - // load the file in two steps. - // First, we load the file's metadata, the structure of which is known. - metadataHolder := stores.SopsFile{} - err := json.Unmarshal(in, &metadataHolder) - if err != nil { - if err, ok := err.(*json.UnmarshalTypeError); ok { - if err.Value == "number" && err.Struct == "Metadata" && err.Field == "version" { - return sops.Tree{}, - fmt.Errorf("SOPS versions higher than 2.0.10 can not automatically decrypt JSON files " + - "created with SOPS 1.x. In order to be able to decrypt this file, you can either edit it " + - "manually and make sure the JSON value under `sops -> version` is a string and not a " + - "number, or you can rotate the file's key with any version of SOPS between 2.0 and 2.0.10 " + - "using `sops -r your_file.json`") - } - } - return sops.Tree{}, fmt.Errorf("Error unmarshalling input json: %s", err) - } - if metadataHolder.Metadata == nil { - return sops.Tree{}, sops.MetadataNotFound - } - metadata, err := metadataHolder.Metadata.ToInternal() + branches, err := store.LoadPlainFile(in) if err != nil { return sops.Tree{}, err } - // After that, we load the whole file into a map. - branch, err := store.treeBranchFromJSON(in) + branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenNone, + }) if err != nil { - return sops.Tree{}, fmt.Errorf("Could not unmarshal input data: %s", err) - } - // Discard metadata, as we already loaded it. - for i, item := range branch { - if item.Key == stores.SopsMetadataKey { - branch = append(branch[:i], branch[i+1:]...) - } + return sops.Tree{}, err } return sops.Tree{ - Branches: sops.TreeBranches{ - branch, - }, + Branches: branches, Metadata: metadata, }, nil } @@ -338,13 +310,13 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { // EmitEncryptedFile returns the encrypted bytes of the json file corresponding to a // sops.Tree runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { - tree := append(in.Branches[0], sops.TreeItem{Key: stores.SopsMetadataKey, Value: stores.MetadataFromInternal(in.Metadata)}) - out, err := store.jsonFromTreeBranch(tree) + branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenNone, + }) if err != nil { - return nil, fmt.Errorf("Error marshaling to json: %s", err) + return nil, fmt.Errorf("Error marshaling metadata: %s", err) } - out = append(out, '\n') - return out, nil + return store.EmitPlainFile(branches) } // EmitPlainFile returns the plaintext bytes of the json file corresponding to a diff --git a/stores/metadata.go b/stores/metadata.go new file mode 100644 index 000000000..c88624d49 --- /dev/null +++ b/stores/metadata.go @@ -0,0 +1,287 @@ +package stores + +import ( + "fmt" + "reflect" + "sort" + "strings" + + "github.com/go-viper/mapstructure/v2" + + "github.com/getsops/sops/v3" +) + +// MetadataFlatten is an enum type +type MetadataFlatten int + +const ( + MetadataFlattenNone MetadataFlatten = iota + MetadataFlattenBelowTop + MetadataFlattenFull +) + +type MetadataOpts struct { + Flatten MetadataFlatten +} + +// SopsPrefix is the prefix for all metadata entry keys. +const SopsPrefix = SopsMetadataKey + "_" + +func sopsToGoMap(mapping sops.TreeBranch) (map[string]interface{}, error) { + result := make(map[string]interface{}) + for _, item := range mapping { + if _, ok := item.Key.(sops.Comment); ok { + continue + } + key, ok := item.Key.(string) + if !ok { + return nil, fmt.Errorf("Unexpected key type %T", item.Key) + } + value, err := sopsToGo(item.Value) + if err != nil { + return nil, err + } + result[key] = value + } + return result, nil +} + +func sopsToGoSlice(slice []interface{}) ([]interface{}, error) { + result := make([]interface{}, len(slice)) + for idx, item := range slice { + if _, ok := item.(sops.Comment); ok { + continue + } + value, err := sopsToGo(item) + if err != nil { + return nil, err + } + result[idx] = value + } + return result, nil +} + +func sopsToGo(value interface{}) (interface{}, error) { + switch value := value.(type) { + case sops.TreeBranch: + return sopsToGoMap(value) + case []interface{}: + return sopsToGoSlice(value) + default: + return value, nil + } +} + +func treeBranchToMetadata(meta sops.TreeBranch) (metadata, error) { + var md metadata + m, err := sopsToGoMap(meta) + if err != nil { + return md, err + } + config := mapstructure.DecoderConfig{ + Result: &md, + WeaklyTypedInput: true, + } + d, err := mapstructure.NewDecoder(&config) + if err != nil { + return md, err + } + err = d.Decode(m) + return md, err +} + +// Extract SOPS metadata from tree branches. +func ExtractMetadata(branches sops.TreeBranches, opts MetadataOpts) (sops.TreeBranches, sops.Metadata, error) { + var metadataTree sops.TreeBranch + if opts.Flatten != MetadataFlattenFull { + first := true + for bi, branch := range branches { + i := 0 + for i < len(branch) { + if branch[i].Key == SopsMetadataKey { + if bi == 0 { + if !first { + return nil, sops.Metadata{}, fmt.Errorf("Found duplicate %v entry", SopsMetadataKey) + } + first = false + if tree, ok := branch[i].Value.(sops.TreeBranch); ok { + metadataTree = tree + } else { + return nil, sops.Metadata{}, fmt.Errorf("Found %v entry that is not a mapping", SopsMetadataKey) + } + } + branch = append(branch[:i], branch[i+1:]...) + } else { + i++ + } + } + branches[bi] = branch + } + } else { + if len(branches) >= 1 { + branch := branches[0] + for i := 0; i < len(branch); i++ { + if key, ok := branch[i].Key.(string); ok { + if strings.HasPrefix(key, SopsPrefix) { + entry := branch[i] + entry.Key = key[len(SopsPrefix):] + metadataTree = append(metadataTree, entry) + branch = append(branch[:i], branch[i+1:]...) + i -= 1 + } + } + } + branches[0] = branch + } + } + if metadataTree == nil { + return nil, sops.Metadata{}, sops.MetadataNotFound + } + if opts.Flatten != MetadataFlattenNone { + var err error + metadataTree, err = unflattenTreeBranch(metadataTree) + if err != nil { + return nil, sops.Metadata{}, err + } + } + md, err := treeBranchToMetadata(metadataTree) + if err != nil { + return nil, sops.Metadata{}, err + } + metadata, err := md.ToInternal() + if err != nil { + return nil, sops.Metadata{}, err + } + return branches, metadata, nil +} + +type mapKey struct { + Name string + Key reflect.Value +} + +// byName implements sort.Interface for []mapKey +type byName []mapKey + +func (mapKeys byName) Len() int { + return len(mapKeys) +} + +func (mapKeys byName) Swap(i, j int) { + mapKeys[i], mapKeys[j] = mapKeys[j], mapKeys[i] +} + +func (mapKeys byName) Less(i, j int) bool { + return mapKeys[i].Name < mapKeys[j].Name +} + +func goToSops(value interface{}) (interface{}, error) { + val := reflect.ValueOf(value) + switch val.Kind() { + case reflect.Array, reflect.Slice: + result := make([]interface{}, val.Len()) + for j := 0; j < val.Len(); j++ { + v, err := goToSops(val.Index(j).Interface()) + if err != nil { + return nil, err + } + result[j] = v + } + return result, nil + case reflect.Map: + keys := val.MapKeys() + sortedKeys := make([]mapKey, len(keys)) + for idx, key := range keys { + sortedKeys[idx] = mapKey{ + Name: key.Interface().(string), + Key: key, + } + } + sort.Sort(byName(sortedKeys)) + result := make(sops.TreeBranch, len(sortedKeys)) + for idx, key := range sortedKeys { + v, err := goToSops(val.MapIndex(key.Key).Interface()) + if err != nil { + return nil, err + } + result[idx] = sops.TreeItem{ + Key: key.Name, + Value: v, + } + } + return result, nil + default: + return value, nil + } +} + +func metadataToTreeBranch(md metadata) (sops.TreeBranch, error) { + var mdMap map[string]interface{} + config := mapstructure.DecoderConfig{ + Result: &mdMap, + } + d, err := mapstructure.NewDecoder(&config) + if err != nil { + return nil, err + } + err = d.Decode(md) + if err != nil { + return nil, err + } + metadata, err := goToSops(mdMap) + if err != nil { + return nil, err + } + if tb, ok := metadata.(sops.TreeBranch); ok { + return tb, nil + } + return nil, fmt.Errorf("Internal error: unexpected metadata conversion result %T", metadata) +} + +func SerializeMetadata(data sops.Tree, opts MetadataOpts) (sops.TreeBranches, error) { + md, err := metadataToTreeBranch(metadataFromInternal(data.Metadata)) + if err != nil { + return nil, fmt.Errorf("Error while serializing metadata: %w", err) + } + if opts.Flatten != MetadataFlattenNone { + var prefix string + if opts.Flatten == MetadataFlattenFull { + prefix = SopsPrefix + } + md, err = flattenTreeBranch(md, prefix) + if err != nil { + return nil, fmt.Errorf("Error while flattening metadata: %w", err) + } + } + if opts.Flatten != MetadataFlattenFull { + md = sops.TreeBranch{ + sops.TreeItem{ + Key: SopsMetadataKey, + Value: md, + }, + } + } + var result sops.TreeBranches + for _, branch := range data.Branches { + newBranch := make(sops.TreeBranch, 0, len(branch)+len(md)) + for _, item := range branch { + if key, ok := item.Key.(string); ok { + if opts.Flatten == MetadataFlattenFull { + if strings.HasPrefix(key, SopsPrefix) { + return nil, fmt.Errorf("Found key %q in encrypted data, which starts with the reserved key prefix %q for SOPS metadata", key, SopsPrefix) + } + } else { + if key == SopsMetadataKey { + return nil, fmt.Errorf("Found key %q in encrypted data, which is a reserved key used for SOPS metadata", key) + } + } + } + newBranch = append(newBranch, item) + } + for _, item := range md { + newBranch = append(newBranch, item) + } + result = append(result, newBranch) + } + return result, nil +} diff --git a/stores/metadata_test.go b/stores/metadata_test.go new file mode 100644 index 000000000..31302392e --- /dev/null +++ b/stores/metadata_test.go @@ -0,0 +1,1010 @@ +package stores + +import ( + "testing" + "time" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/age" + "github.com/getsops/sops/v3/azkv" + "github.com/getsops/sops/v3/gcpkms" + "github.com/getsops/sops/v3/hckms" + "github.com/getsops/sops/v3/hcvault" + "github.com/getsops/sops/v3/kms" + "github.com/getsops/sops/v3/pgp" + "github.com/stretchr/testify/assert" +) + +func TestExtractMetadata(t *testing.T) { + var ( + broken1 = sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + } + broken2 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: "foo", + }, + } + empty = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{}, + }, + } + + minimal1 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "LastModified", + Value: "2025-03-23T10:20:30Z", + }, + }, + }, + } + minimal2 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + + multiple = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "unencrypted_suffix", + Value: "foo", + }, + sops.TreeItem{ + Key: "encrypted_suffix", + Value: "bar", + }, + sops.TreeItem{ + Key: "unencrypted_regex", + Value: "baz", + }, + sops.TreeItem{ + Key: "encrypted_regex", + Value: "bam", + }, + sops.TreeItem{ + Key: "unencrypted_comment_regex", + Value: "foobar", + }, + sops.TreeItem{ + Key: "encrypted_comment_regex", + Value: "bazbam", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + + single1 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "unencrypted_suffix", + Value: "foo", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + single2 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "encrypted_suffix", + Value: "bar", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + single3 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "unencrypted_regex", + Value: "baz", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + single4 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "encrypted_regex", + Value: "bam", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + single5 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "unencrypted_comment_regex", + Value: "foobar", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + single6 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "encrypted_comment_regex", + Value: "bazbam", + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD", + }, + sops.TreeItem{ + Key: "fp", + Value: "1234", + }, + }, + }, + }, + }, + }, + } + + completeKeyGroup = sops.TreeBranch{ + sops.TreeItem{ + Key: "kms", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD AWS KMS (inner)", + }, + sops.TreeItem{ + Key: "arn", + Value: "AWS KMS ARN (inner)", + }, + sops.TreeItem{ + Key: "role", + Value: "AWS KMS role (inner)", + }, + sops.TreeItem{ + Key: "context", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + sops.TreeItem{ + Key: "aws_profile", + Value: "AWS KMS profile (inner)", + }, + }, + }, + }, + sops.TreeItem{ + Key: "gcp_kms", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD GCP KMS (inner)", + }, + sops.TreeItem{ + Key: "resource_id", + Value: "GCP KMS resource ID (inner)", + }, + }, + }, + }, + sops.TreeItem{ + Key: "hckms", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD HC KMS (inner)", + }, + sops.TreeItem{ + Key: "key_id", + Value: "HC KMS (inner):key ID (inner)", + }, + }, + }, + }, + sops.TreeItem{ + Key: "azure_kv", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD AZKV (inner)", + }, + sops.TreeItem{ + Key: "vault_url", + Value: "AZKV vault URL (inner)", + }, + sops.TreeItem{ + Key: "name", + Value: "AZKV name (inner)", + }, + sops.TreeItem{ + Key: "version", + Value: "AZKV version (inner)", + }, + }, + }, + }, + sops.TreeItem{ + Key: "hc_vault", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD HC Vault (inner)", + }, + sops.TreeItem{ + Key: "vault_address", + Value: "HC Vault address (inner)", + }, + sops.TreeItem{ + Key: "engine_path", + Value: "HC Vault engine path (inner)", + }, + sops.TreeItem{ + Key: "key_name", + Value: "HC Vault key name (inner)", + }, + }, + }, + }, + sops.TreeItem{ + Key: "age", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "recipient", + Value: "age recipient (inner)", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD age (inner)", + }, + }, + }, + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD PGP (inner)", + }, + sops.TreeItem{ + Key: "fp", + Value: "PGP fingerprint (inner)", + }, + }, + }, + }, + } + + everything1 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "shamir_threshold", + Value: 2, + }, + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "mac", + Value: "asdf", + }, + sops.TreeItem{ + Key: "encrypted_comment_regex", + Value: "bazbam", + }, + sops.TreeItem{ + Key: "mac_only_encrypted", + Value: true, + }, + sops.TreeItem{ + Key: "version", + Value: "barbaz", + }, + sops.TreeItem{ + Key: "kms", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD AWS KMS", + }, + sops.TreeItem{ + Key: "arn", + Value: "AWS KMS ARN", + }, + sops.TreeItem{ + Key: "role", + Value: "AWS KMS role", + }, + sops.TreeItem{ + Key: "context", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + sops.TreeItem{ + Key: "aws_profile", + Value: "AWS KMS profile", + }, + }, + }, + }, + sops.TreeItem{ + Key: "gcp_kms", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD GCP KMS", + }, + sops.TreeItem{ + Key: "resource_id", + Value: "GCP KMS resource ID", + }, + }, + }, + }, + sops.TreeItem{ + Key: "hckms", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD HC KMS", + }, + sops.TreeItem{ + Key: "key_id", + Value: "HC KMS:key ID", + }, + }, + }, + }, + sops.TreeItem{ + Key: "azure_kv", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD AZKV", + }, + sops.TreeItem{ + Key: "vault_url", + Value: "AZKV vault URL", + }, + sops.TreeItem{ + Key: "name", + Value: "AZKV name", + }, + sops.TreeItem{ + Key: "version", + Value: "AZKV version", + }, + }, + }, + }, + sops.TreeItem{ + Key: "hc_vault", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD HC Vault", + }, + sops.TreeItem{ + Key: "vault_address", + Value: "HC Vault address", + }, + sops.TreeItem{ + Key: "engine_path", + Value: "HC Vault engine path", + }, + sops.TreeItem{ + Key: "key_name", + Value: "HC Vault key name", + }, + }, + }, + }, + sops.TreeItem{ + Key: "age", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "recipient", + Value: "age recipient", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD age", + }, + }, + }, + }, + sops.TreeItem{ + Key: "pgp", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "created_at", + Value: "2025-03-23T10:20:29Z", + }, + sops.TreeItem{ + Key: "enc", + Value: "ABCD PGP", + }, + sops.TreeItem{ + Key: "fp", + Value: "PGP fingerprint", + }, + }, + }, + }, + // This will be ignored: + sops.TreeItem{ + Key: "key_groups", + Value: []interface{}{ + completeKeyGroup, + }, + }, + }, + }, + } + everything2 = sops.TreeBranch{ + sops.TreeItem{ + Key: "sops", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "shamir_threshold", + Value: 2, + }, + sops.TreeItem{ + Key: "lastmodified", + Value: "2025-03-23T10:20:30Z", + }, + sops.TreeItem{ + Key: "mac", + Value: "asdf", + }, + sops.TreeItem{ + Key: "encrypted_comment_regex", + Value: "bazbam", + }, + sops.TreeItem{ + Key: "mac_only_encrypted", + Value: true, + }, + sops.TreeItem{ + Key: "version", + Value: "barbaz", + }, + sops.TreeItem{ + Key: "key_groups", + Value: []interface{}{ + completeKeyGroup, + }, + }, + }, + }, + } + ) + + branches, metadata, err := ExtractMetadata([]sops.TreeBranch{broken1}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.NotNil(t, err) + assert.Equal(t, sops.MetadataNotFound, err) + assert.Nil(t, branches) + assert.Equal(t, sops.Metadata{}, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{broken2}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.NotNil(t, err) + assert.Equal(t, "Found sops entry that is not a mapping", err.Error()) + assert.Nil(t, branches) + assert.Equal(t, sops.Metadata{}, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{empty}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.NotNil(t, err) + assert.Equal(t, "parsing time \"\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"\" as \"2006\"", err.Error()) + assert.Nil(t, branches) + assert.Equal(t, sops.Metadata{}, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{minimal1}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.NotNil(t, err) + assert.Equal(t, "No keys found in file", err.Error()) + assert.Nil(t, branches) + assert.Equal(t, sops.Metadata{}, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{minimal2}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + UnencryptedSuffix: "_unencrypted", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{multiple}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.NotNil(t, err) + assert.Equal(t, "Cannot use more than one of encrypted_suffix, unencrypted_suffix, encrypted_regex, unencrypted_regex, encrypted_comment_regex, or unencrypted_comment_regex in the same file", err.Error()) + assert.Nil(t, branches) + assert.Equal(t, sops.Metadata{}, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{single1}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + UnencryptedSuffix: "foo", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{single2}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + EncryptedSuffix: "bar", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{single3}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + UnencryptedRegex: "baz", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{single4}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + EncryptedRegex: "bam", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{single5}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + UnencryptedCommentRegex: "foobar", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{single6}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + LastModified: time.Unix(1742725230, 0).UTC(), + EncryptedCommentRegex: "bazbam", + KeyGroups: []sops.KeyGroup{ + { + &pgp.MasterKey{ + Fingerprint: "1234", + EncryptedKey: "ABCD", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{everything1}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + bar := "bar" + assert.Equal(t, sops.Metadata{ + ShamirThreshold: 2, + LastModified: time.Unix(1742725230, 0).UTC(), + MessageAuthenticationCode: "asdf", + EncryptedCommentRegex: "bazbam", + MACOnlyEncrypted: true, + Version: "barbaz", + KeyGroups: []sops.KeyGroup{ + { + &kms.MasterKey{ + Arn: "AWS KMS ARN", + Role: "AWS KMS role", + EncryptedKey: "ABCD AWS KMS", + CreationDate: time.Unix(1742725229, 0).UTC(), + EncryptionContext: map[string]*string{ + "foo": &bar, + }, + AwsProfile: "AWS KMS profile", + }, + &gcpkms.MasterKey{ + ResourceID: "GCP KMS resource ID", + EncryptedKey: "ABCD GCP KMS", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &hckms.MasterKey{ + KeyID: "HC KMS:key ID", + Region: "HC KMS", + KeyUUID: "key ID", + EncryptedKey: "ABCD HC KMS", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &azkv.MasterKey{ + VaultURL: "AZKV vault URL", + Name: "AZKV name", + Version: "AZKV version", + EncryptedKey: "ABCD AZKV", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &hcvault.MasterKey{ + VaultAddress: "HC Vault address", + EnginePath: "HC Vault engine path", + KeyName: "HC Vault key name", + EncryptedKey: "ABCD HC Vault", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &pgp.MasterKey{ + Fingerprint: "PGP fingerprint", + EncryptedKey: "ABCD PGP", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &age.MasterKey{ + Recipient: "age recipient", + EncryptedKey: "ABCD age", + }, + }, + }, + }, metadata) + + branches, metadata, err = ExtractMetadata([]sops.TreeBranch{everything2}, MetadataOpts{Flatten: MetadataFlattenNone}) + assert.Nil(t, err) + assert.Equal(t, sops.Metadata{ + ShamirThreshold: 2, + LastModified: time.Unix(1742725230, 0).UTC(), + MessageAuthenticationCode: "asdf", + EncryptedCommentRegex: "bazbam", + MACOnlyEncrypted: true, + Version: "barbaz", + KeyGroups: []sops.KeyGroup{ + { + &kms.MasterKey{ + Arn: "AWS KMS ARN (inner)", + Role: "AWS KMS role (inner)", + EncryptedKey: "ABCD AWS KMS (inner)", + CreationDate: time.Unix(1742725229, 0).UTC(), + EncryptionContext: map[string]*string{ + "foo": &bar, + }, + AwsProfile: "AWS KMS profile (inner)", + }, + &gcpkms.MasterKey{ + ResourceID: "GCP KMS resource ID (inner)", + EncryptedKey: "ABCD GCP KMS (inner)", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &hckms.MasterKey{ + KeyID: "HC KMS (inner):key ID (inner)", + Region: "HC KMS (inner)", + KeyUUID: "key ID (inner)", + EncryptedKey: "ABCD HC KMS (inner)", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &azkv.MasterKey{ + VaultURL: "AZKV vault URL (inner)", + Name: "AZKV name (inner)", + Version: "AZKV version (inner)", + EncryptedKey: "ABCD AZKV (inner)", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &hcvault.MasterKey{ + VaultAddress: "HC Vault address (inner)", + EnginePath: "HC Vault engine path (inner)", + KeyName: "HC Vault key name (inner)", + EncryptedKey: "ABCD HC Vault (inner)", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &pgp.MasterKey{ + Fingerprint: "PGP fingerprint (inner)", + EncryptedKey: "ABCD PGP (inner)", + CreationDate: time.Unix(1742725229, 0).UTC(), + }, + &age.MasterKey{ + Recipient: "age recipient (inner)", + EncryptedKey: "ABCD age (inner)", + }, + }, + }, + }, metadata) +} + +func TestSerializeMetadata(t *testing.T) { +} diff --git a/stores/stores.go b/stores/stores.go index 11e362a5d..1b683714a 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -30,102 +30,93 @@ const ( SopsMetadataKey = "sops" ) -// SopsFile is a struct used by the stores as a helper to unmarshal the SOPS metadata -type SopsFile struct { - // Metadata is a pointer so we can easily tell when the field is not present - // in the SOPS file by checking for nil. This way we can show the user a - // helpful error message indicating that the metadata wasn't found, instead - // of showing a cryptic parsing error - Metadata *Metadata `yaml:"sops" json:"sops" ini:"sops"` -} - // Metadata is stored in SOPS encrypted files, and it contains the information necessary to decrypt the file. // This struct is just used for serialization, and SOPS uses another struct internally, sops.Metadata. It exists // in order to allow the binary format to stay backwards compatible over time, but at the same time allow the internal // representation SOPS uses to change over time. -type Metadata struct { - ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty"` - KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty"` - KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"` - GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` - HCKmsKeys []hckmskey `yaml:"hckms,omitempty" json:"hckms,omitempty"` - AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` - VaultKeys []vaultkey `yaml:"hc_vault,omitempty" json:"hc_vault,omitempty"` - AgeKeys []agekey `yaml:"age,omitempty" json:"age,omitempty"` - LastModified string `yaml:"lastmodified" json:"lastmodified"` - MessageAuthenticationCode string `yaml:"mac" json:"mac"` - PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"` - UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"` - EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"` - UnencryptedRegex string `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty"` - EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"` - UnencryptedCommentRegex string `yaml:"unencrypted_comment_regex,omitempty" json:"unencrypted_comment_regex,omitempty"` - EncryptedCommentRegex string `yaml:"encrypted_comment_regex,omitempty" json:"encrypted_comment_regex,omitempty"` - MACOnlyEncrypted bool `yaml:"mac_only_encrypted,omitempty" json:"mac_only_encrypted,omitempty"` - Version string `yaml:"version" json:"version"` +type metadata struct { + ShamirThreshold int `mapstructure:"shamir_threshold,omitempty"` + KeyGroups []keygroup `mapstructure:"key_groups,omitempty,deep"` + KMSKeys []kmskey `mapstructure:"kms,omitempty,deep"` + GCPKMSKeys []gcpkmskey `mapstructure:"gcp_kms,omitempty,deep"` + HCKmsKeys []hckmskey `mapstructure:"hckms,omitempty,deep"` + AzureKeyVaultKeys []azkvkey `mapstructure:"azure_kv,omitempty,deep"` + VaultKeys []vaultkey `mapstructure:"hc_vault,omitempty,deep"` + AgeKeys []agekey `mapstructure:"age,omitempty,deep"` + LastModified string `mapstructure:"lastmodified"` + MessageAuthenticationCode string `mapstructure:"mac"` + PGPKeys []pgpkey `mapstructure:"pgp,omitempty,deep"` + UnencryptedSuffix string `mapstructure:"unencrypted_suffix,omitempty"` + EncryptedSuffix string `mapstructure:"encrypted_suffix,omitempty"` + UnencryptedRegex string `mapstructure:"unencrypted_regex,omitempty"` + EncryptedRegex string `mapstructure:"encrypted_regex,omitempty"` + UnencryptedCommentRegex string `mapstructure:"unencrypted_comment_regex,omitempty"` + EncryptedCommentRegex string `mapstructure:"encrypted_comment_regex,omitempty"` + MACOnlyEncrypted bool `mapstructure:"mac_only_encrypted,omitempty"` + Version string `mapstructure:"version"` } type keygroup struct { - PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"` - KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"` - GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` - HCKmsKeys []hckmskey `yaml:"hckms,omitempty" json:"hckms,omitempty"` - AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` - VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"` - AgeKeys []agekey `yaml:"age" json:"age"` + PGPKeys []pgpkey `mapstructure:"pgp,omitempty,deep"` + KMSKeys []kmskey `mapstructure:"kms,omitempty,deep"` + GCPKMSKeys []gcpkmskey `mapstructure:"gcp_kms,omitempty,deep"` + HCKmsKeys []hckmskey `mapstructure:"hckms,omitempty,deep"` + AzureKeyVaultKeys []azkvkey `mapstructure:"azure_kv,omitempty,deep"` + VaultKeys []vaultkey `mapstructure:"hc_vault,deep"` + AgeKeys []agekey `mapstructure:"age,deep"` } type pgpkey struct { - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` - Fingerprint string `yaml:"fp" json:"fp"` + CreatedAt string `mapstructure:"created_at"` + EncryptedDataKey string `mapstructure:"enc"` + Fingerprint string `mapstructure:"fp"` } type kmskey struct { - Arn string `yaml:"arn" json:"arn"` - Role string `yaml:"role,omitempty" json:"role,omitempty"` - Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` - AwsProfile string `yaml:"aws_profile" json:"aws_profile"` + Arn string `mapstructure:"arn"` + Role string `mapstructure:"role,omitempty"` + Context map[string]*string `mapstructure:"context,omitempty"` + CreatedAt string `mapstructure:"created_at"` + EncryptedDataKey string `mapstructure:"enc"` + AwsProfile string `mapstructure:"aws_profile"` } type gcpkmskey struct { - ResourceID string `yaml:"resource_id" json:"resource_id"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + ResourceID string `mapstructure:"resource_id"` + CreatedAt string `mapstructure:"created_at"` + EncryptedDataKey string `mapstructure:"enc"` } type vaultkey struct { - VaultAddress string `yaml:"vault_address" json:"vault_address"` - EnginePath string `yaml:"engine_path" json:"engine_path"` - KeyName string `yaml:"key_name" json:"key_name"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + VaultAddress string `mapstructure:"vault_address"` + EnginePath string `mapstructure:"engine_path"` + KeyName string `mapstructure:"key_name"` + CreatedAt string `mapstructure:"created_at"` + EncryptedDataKey string `mapstructure:"enc"` } type azkvkey struct { - VaultURL string `yaml:"vault_url" json:"vault_url"` - Name string `yaml:"name" json:"name"` - Version string `yaml:"version" json:"version"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + VaultURL string `mapstructure:"vault_url"` + Name string `mapstructure:"name"` + Version string `mapstructure:"version"` + CreatedAt string `mapstructure:"created_at"` + EncryptedDataKey string `mapstructure:"enc"` } type agekey struct { - Recipient string `yaml:"recipient" json:"recipient"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + Recipient string `mapstructure:"recipient"` + EncryptedDataKey string `mapstructure:"enc"` } type hckmskey struct { - KeyID string `yaml:"key_id" json:"key_id"` - CreatedAt string `yaml:"created_at" json:"created_at"` - EncryptedDataKey string `yaml:"enc" json:"enc"` + KeyID string `mapstructure:"key_id"` + CreatedAt string `mapstructure:"created_at"` + EncryptedDataKey string `mapstructure:"enc"` } // MetadataFromInternal converts an internal SOPS metadata representation to a representation appropriate for storage -func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { - var m Metadata +func metadataFromInternal(sopsMetadata sops.Metadata) metadata { + var m metadata m.LastModified = sopsMetadata.LastModified.Format(time.RFC3339) m.UnencryptedSuffix = sopsMetadata.UnencryptedSuffix m.EncryptedSuffix = sopsMetadata.EncryptedSuffix @@ -267,7 +258,7 @@ func hckmsKeysFromGroup(group sops.KeyGroup) (keys []hckmskey) { } // ToInternal converts a storage-appropriate Metadata struct to a SOPS internal representation -func (m *Metadata) ToInternal() (sops.Metadata, error) { +func (m *metadata) ToInternal() (sops.Metadata, error) { lastModified, err := time.Parse(time.RFC3339, m.LastModified) if err != nil { return sops.Metadata{}, err @@ -374,7 +365,7 @@ func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmske return internalGroup, nil } -func (m *Metadata) internalKeygroups() ([]sops.KeyGroup, error) { +func (m *metadata) internalKeygroups() ([]sops.KeyGroup, error) { var internalGroups []sops.KeyGroup if len(m.PGPKeys) > 0 || len(m.KMSKeys) > 0 || len(m.GCPKMSKeys) > 0 || len(m.HCKmsKeys) > 0 || len(m.AzureKeyVaultKeys) > 0 || len(m.VaultKeys) > 0 || len(m.AgeKeys) > 0 { internalGroup, err := internalGroupFrom(m.KMSKeys, m.PGPKeys, m.GCPKMSKeys, m.HCKmsKeys, m.AzureKeyVaultKeys, m.VaultKeys, m.AgeKeys) @@ -613,3 +604,23 @@ func ValToString(v interface{}) string { return fmt.Sprintf("%v", v) } } + +// DecodeNewLines replaces \\n with \n for all string values in the map. +// Used by config stores that do not handle multi-line values (ini, dotenv). +func DecodeNewLines(m map[string]interface{}) { + for k, v := range m { + if s, ok := v.(string); ok { + m[k] = strings.Replace(s, "\\n", "\n", -1) + } + } +} + +// EncodeNewLines replaces \n with \\n for all string values in the map. +// Used by config stores that do not handle multi-line values (ini, dotenv). +func EncodeNewLines(m map[string]interface{}) { + for k, v := range m { + if s, ok := v.(string); ok { + m[k] = strings.Replace(s, "\n", "\\n", -1) + } + } +} diff --git a/stores/stores_test.go b/stores/stores_test.go index 31ced210a..3ded02332 100644 --- a/stores/stores_test.go +++ b/stores/stores_test.go @@ -25,3 +25,37 @@ func TestValToString(t *testing.T) { assert.Equal(t, "2025-01-02T03:04:05Z", ValToString(ts)) assert.Equal(t, "a string", ValToString("a string")) } + +func TestDecodeNewLines(t *testing.T) { + tests := []struct { + input map[string]interface{} + want map[string]interface{} + }{ + {map[string]interface{}{"mac": "line1\\nline2"}, map[string]interface{}{"mac": "line1\nline2"}}, + {map[string]interface{}{"mac": "line1\\n\\n\\nline2\\n\\nline3"}, map[string]interface{}{"mac": "line1\n\n\nline2\n\nline3"}}, + } + + for _, tt := range tests { + DecodeNewLines(tt.input) + for k, v := range tt.want { + assert.Equal(t, v, tt.input[k]) + } + } +} + +func TestEncodeNewLines(t *testing.T) { + tests := []struct { + input map[string]interface{} + want map[string]interface{} + }{ + {map[string]interface{}{"mac": "line1\nline2"}, map[string]interface{}{"mac": "line1\\nline2"}}, + {map[string]interface{}{"mac": "line1\n\n\nline2\n\nline3"}, map[string]interface{}{"mac": "line1\\n\\n\\nline2\\n\\nline3"}}, + } + + for _, tt := range tests { + EncodeNewLines(tt.input) + for k, v := range tt.want { + assert.Equal(t, v, tt.input[k]) + } + } +} diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 4d48e2c48..1589fdcdc 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -302,45 +302,16 @@ func (store *Store) appendTreeBranch(branch sops.TreeBranch, mapping *yaml.Node) // LoadEncryptedFile loads the contents of an encrypted yaml file onto a // sops.Tree runtime object func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { - // Because we don't know what fields the input file will have, we have to - // load the file in two steps. - // First, we load the file's metadata, the structure of which is known. - metadataHolder := stores.SopsFile{} - err := yaml.Unmarshal(in, &metadataHolder) + branches, err := store.LoadPlainFile(in) if err != nil { - return sops.Tree{}, fmt.Errorf("Error unmarshalling input yaml: %s", err) - } - if metadataHolder.Metadata == nil { - return sops.Tree{}, sops.MetadataNotFound + return sops.Tree{}, err } - metadata, err := metadataHolder.Metadata.ToInternal() + branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenNone, + }) if err != nil { return sops.Tree{}, err } - var branches sops.TreeBranches - d := yaml.NewDecoder(bytes.NewReader(in)) - for { - var data yaml.Node - err := d.Decode(&data) - if err == io.EOF { - break - } - if err != nil { - return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err) - } - - branch, err := store.yamlDocumentNodeToTreeBranch(data) - if err != nil { - return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err) - } - - for i, elt := range branch { - if elt.Key == stores.SopsMetadataKey { // Erase - branch = append(branch[:i], branch[i+1:]...) - } - } - branches = append(branches, branch) - } return sops.Tree{ Branches: branches, Metadata: metadata, @@ -390,37 +361,13 @@ func (store *Store) getIndentation() (int, error) { // EmitEncryptedFile returns the encrypted bytes of the yaml file corresponding to a // sops.Tree runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { - var b bytes.Buffer - e := yaml.NewEncoder(io.Writer(&b)) - indent, err := store.getIndentation() + branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{ + Flatten: stores.MetadataFlattenNone, + }) if err != nil { - return nil, err + return nil, fmt.Errorf("Error marshaling metadata: %s", err) } - e.SetIndent(indent) - for _, branch := range in.Branches { - // Document root - var doc = yaml.Node{} - doc.Kind = yaml.DocumentNode - // Add global mapping - var mapping = yaml.Node{} - mapping.Kind = yaml.MappingNode - doc.Content = append(doc.Content, &mapping) - // Create copy of branch with metadata appended - branch = append(sops.TreeBranch(nil), branch...) - branch = append(branch, sops.TreeItem{ - Key: stores.SopsMetadataKey, - Value: stores.MetadataFromInternal(in.Metadata), - }) - // Marshal branch to global mapping node - store.appendTreeBranch(branch, &mapping) - // Encode YAML - err := e.Encode(&doc) - if err != nil { - return nil, fmt.Errorf("Error marshaling to yaml: %s", err) - } - } - e.Close() - return b.Bytes(), nil + return store.EmitPlainFile(branches) } // EmitPlainFile returns the plaintext bytes of the yaml file corresponding to a @@ -446,7 +393,7 @@ func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { // Encode YAML err := e.Encode(&doc) if err != nil { - return nil, fmt.Errorf("Error marshaling to yaml: %s", err) + return nil, fmt.Errorf("Error marshaling to YAML: %s", err) } } e.Close()