Add Atmos manifest lists merge strategies (#609)

* updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates
cloudposse · May 27, 2024 · 3e0983a · 3e0983a
1 parent d00aa78
commit 3e0983a
Show file tree

Hide file tree

Showing 21 changed files with 459 additions and 145 deletions.
diff --git a/atmos.yaml b/atmos.yaml
@@ -68,7 +68,7 @@ stacks:
   name_pattern: "{tenant}-{environment}-{stage}"
 
 workflows:
-  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
+  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
   # Supports both absolute and relative paths
   base_path: "stacks/workflows"
 
@@ -301,12 +301,12 @@ integrations:
 schemas:
   # https://json-schema.org
   jsonschema:
-    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/jsonschema"
   # https://www.openpolicyagent.org
   opa:
-    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/opa"
   # JSON Schema to validate Atmos manifests
@@ -318,7 +318,7 @@ schemas:
   # https://www.schemastore.org/json
   # https://github.com/SchemaStore/schemastore
   atmos:
-    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
     # Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
     manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
 
@@ -338,3 +338,16 @@ templates:
       timeout: 5
       # https://docs.gomplate.ca/datasources
       datasources: {}
+
+settings:
+  # `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
+  # Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
+  # The following strategies are supported:
+  # `replace`: Most recent list imported wins (the default behavior).
+  # `append`:  The sequence of lists is appended in the same order as imports.
+  # `merge`:   The items in the destination list are deep-merged with the items in the source list.
+  #            The items in the source list take precedence.
+  #            The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
+  #            If the source and destination lists have the same length, all items in the destination lists are
+  #            deep-merged with all items in the source list.
+  list_merge_strategy: replace
diff --git a/examples/demo-stacks/atmos.yaml b/examples/demo-stacks/atmos.yaml
@@ -17,5 +17,5 @@ stacks:
   name_pattern: "{stage}"
 
 logs:
-  file: "/dev/stdout"
+  file: "/dev/stderr"
   level: Info
diff --git a/examples/quick-start/Dockerfile b/examples/quick-start/Dockerfile
@@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian
 # https://atmos.tools/
 # https://github.com/cloudposse/atmos
 # https://github.com/cloudposse/atmos/releases
-ARG ATMOS_VERSION=1.75.0
+ARG ATMOS_VERSION=1.76.0
 
 # Terraform: https://github.com/hashicorp/terraform/releases
 ARG TF_VERSION=1.8.4

diff --git a/examples/quick-start/atmos.yaml b/examples/quick-start/atmos.yaml
@@ -68,7 +68,7 @@ stacks:
   name_pattern: "{tenant}-{environment}-{stage}"
 
 workflows:
-  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
+  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
   # Supports both absolute and relative paths
   base_path: "stacks/workflows"
 
@@ -234,12 +234,12 @@ commands:
 schemas:
   # https://json-schema.org
   jsonschema:
-    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/jsonschema"
   # https://www.openpolicyagent.org
   opa:
-    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/opa"
   # JSON Schema to validate Atmos manifests
@@ -251,7 +251,7 @@ schemas:
   # https://www.schemastore.org/json
   # https://github.com/SchemaStore/schemastore
   atmos:
-    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
     # Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
     manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
 
@@ -270,3 +270,16 @@ templates:
       enabled: true
       # https://docs.gomplate.ca/datasources
       datasources: {}
+
+settings:
+  # `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
+  # Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
+  # The following strategies are supported:
+  # `replace`: Most recent list imported wins (the default behavior).
+  # `append`:  The sequence of lists is appended in the same order as imports.
+  # `merge`:   The items in the destination list are deep-merged with the items in the source list.
+  #            The items in the source list take precedence.
+  #            The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
+  #            If the source and destination lists have the same length, all items in the destination lists are
+  #            deep-merged with all items in the source list.
+  list_merge_strategy: replace
diff --git a/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml b/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml
@@ -68,7 +68,7 @@ stacks:
   name_pattern: "{tenant}-{environment}-{stage}"
 
 workflows:
-  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
+  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
   # Supports both absolute and relative paths
   base_path: "stacks/workflows"
 
@@ -235,12 +235,12 @@ commands:
 schemas:
   # https://json-schema.org
   jsonschema:
-    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/jsonschema"
   # https://www.openpolicyagent.org
   opa:
-    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/opa"
   # JSON Schema to validate Atmos manifests
@@ -252,7 +252,7 @@ schemas:
   # https://www.schemastore.org/json
   # https://github.com/SchemaStore/schemastore
   atmos:
-    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
     # Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
     manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
 
@@ -270,3 +270,16 @@ templates:
       enabled: true
       # https://docs.gomplate.ca/datasources
       datasources: {}
+
+settings:
+  # `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
+  # Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
+  # The following strategies are supported:
+  # `replace`: Most recent list imported wins (the default behavior).
+  # `append`:  The sequence of lists is appended in the same order as imports.
+  # `merge`:   The items in the destination list are deep-merged with the items in the source list.
+  #            The items in the source list take precedence.
+  #            The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
+  #            If the source and destination lists have the same length, all items in the destination lists are
+  #            deep-merged with all items in the source list.
+  list_merge_strategy: replace
diff --git a/examples/tests/atmos.yaml b/examples/tests/atmos.yaml
@@ -75,7 +75,7 @@ stacks:
   name_template: "{{.vars.tenant}}-{{.vars.environment}}-{{.vars.stage}}"
 
 workflows:
-  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
+  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
   # Supports both absolute and relative paths
   base_path: "stacks/workflows"
 
@@ -328,12 +328,12 @@ integrations:
 schemas:
   # https://json-schema.org
   jsonschema:
-    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/jsonschema"
   # https://www.openpolicyagent.org
   opa:
-    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/opa"
   # JSON Schema to validate Atmos manifests
@@ -345,7 +345,7 @@ schemas:
   # https://www.schemastore.org/json
   # https://github.com/SchemaStore/schemastore
   atmos:
-    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
     # Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
     manifest: "../quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
 
@@ -378,3 +378,16 @@ templates:
       timeout: 5
       # https://docs.gomplate.ca/datasources
       datasources: {}
+
+settings:
+  # `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
+  # Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
+  # The following strategies are supported:
+  # `replace`: Most recent list imported wins (the default behavior).
+  # `append`:  The sequence of lists is appended in the same order as imports.
+  # `merge`:   The items in the destination list are deep-merged with the items in the source list.
+  #            The items in the source list take precedence.
+  #            The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
+  #            If the source and destination lists have the same length, all items in the destination lists are
+  #            deep-merged with all items in the source list.
+  list_merge_strategy: replace
diff --git a/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml b/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml
@@ -70,7 +70,7 @@ stacks:
   name_template: "{{.vars.tenant}}-{{.vars.environment}}-{{.vars.stage}}"
 
 workflows:
-  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
+  # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
   # Supports both absolute and relative paths
   base_path: "stacks/workflows"
 
@@ -576,12 +576,12 @@ integrations:
 schemas:
   # https://json-schema.org
   jsonschema:
-    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/jsonschema"
   # https://www.openpolicyagent.org
   opa:
-    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
     # Supports both absolute and relative paths
     base_path: "stacks/schemas/opa"
   # JSON Schema to validate Atmos manifests
@@ -593,7 +593,7 @@ schemas:
   # https://www.schemastore.org/json
   # https://github.com/SchemaStore/schemastore
   atmos:
-    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
+    # Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
     # Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
     manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
 
@@ -625,3 +625,16 @@ templates:
       timeout: 5
       # https://docs.gomplate.ca/datasources
       datasources: {}
+
+settings:
+  # `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
+  # Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
+  # The following strategies are supported:
+  # `replace`: Most recent list imported wins (the default behavior).
+  # `append`:  The sequence of lists is appended in the same order as imports.
+  # `merge`:   The items in the destination list are deep-merged with the items in the source list.
+  #            The items in the source list take precedence.
+  #            The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
+  #            If the source and destination lists have the same length, all items in the destination lists are
+  #            deep-merged with all items in the source list.
+  list_merge_strategy: replace
diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module github.com/cloudposse/atmos
 go 1.21
 
 require (
+	dario.cat/mergo v1.0.0
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/chroma v0.10.0
 	github.com/arsham/figurine v1.3.0
@@ -21,7 +22,6 @@ require (
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/hcl/v2 v2.20.1
 	github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30
-	github.com/imdario/mergo v0.3.13
 	github.com/ivanpirog/coloredcobra v1.0.1
 	github.com/json-iterator/go v1.1.12
 	github.com/jwalton/go-supportscolor v1.2.0
@@ -48,7 +48,6 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/storage v1.36.0 // indirect
-	dario.cat/mergo v1.0.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
@@ -155,6 +154,7 @@ require (
 	github.com/hashicorp/vault/sdk v0.5.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

diff --git a/internal/exec/utils.go b/internal/exec/utils.go
@@ -216,6 +216,7 @@ func processCommandLineArgs(
 	configAndStacksInfo.RedirectStdErr = argsAndFlagsInfo.RedirectStdErr
 	configAndStacksInfo.LogsLevel = argsAndFlagsInfo.LogsLevel
 	configAndStacksInfo.LogsFile = argsAndFlagsInfo.LogsFile
+	configAndStacksInfo.SettingsListMergeStrategy = argsAndFlagsInfo.SettingsListMergeStrategy
 
 	// Check if `-h` or `--help` flags are specified
 	if argsAndFlagsInfo.NeedHelp {
@@ -885,6 +886,19 @@ func processArgsAndFlags(componentType string, inputArgsAndFlags []string) (sche
 			info.LogsFile = logsFileFlagParts[1]
 		}
 
+		if arg == cfg.SettingsListMergeStrategyFlag {
+			if len(inputArgsAndFlags) <= (i + 1) {
+				return info, fmt.Errorf("invalid flag: %s", arg)
+			}
+			info.SettingsListMergeStrategy = inputArgsAndFlags[i+1]
+		} else if strings.HasPrefix(arg+"=", cfg.SettingsListMergeStrategyFlag) {
+			var settingsListMergeStrategyParts = strings.Split(arg, "=")
+			if len(settingsListMergeStrategyParts) != 2 {
+				return info, fmt.Errorf("invalid flag: %s", arg)
+			}
+			info.SettingsListMergeStrategy = settingsListMergeStrategyParts[1]
+		}
+
 		if arg == cfg.FromPlanFlag {
 			info.UseTerraformPlan = true
 		}

diff --git a/pkg/config/const.go b/pkg/config/const.go
@@ -58,4 +58,6 @@ const (
 
 	LogsLevelFlag = "--logs-level"
 	LogsFileFlag  = "--logs-file"
+
+	SettingsListMergeStrategyFlag = "--settings-list-merge-strategy"
 )
diff --git a/pkg/config/utils.go b/pkg/config/utils.go
@@ -320,6 +320,12 @@ func processEnvVars(cliConfig *schema.CliConfiguration) error {
 		cliConfig.Logs.Level = logsLevel
 	}
 
+	listMergeStrategy := os.Getenv("ATMOS_SETTINGS_LIST_MERGE_STRATEGY")
+	if len(listMergeStrategy) > 0 {
+		u.LogTrace(*cliConfig, fmt.Sprintf("Found ENV var ATMOS_SETTINGS_LIST_MERGE_STRATEGY=%s", listMergeStrategy))
+		cliConfig.Settings.ListMergeStrategy = listMergeStrategy
+	}
+
 	return nil
 }
 
@@ -416,6 +422,10 @@ func processCommandLineArgs(cliConfig *schema.CliConfiguration, configAndStacksI
 		cliConfig.Logs.File = configAndStacksInfo.LogsFile
 		u.LogTrace(*cliConfig, fmt.Sprintf("Using command line argument '%s=%s'", LogsFileFlag, configAndStacksInfo.LogsFile))
 	}
+	if len(configAndStacksInfo.SettingsListMergeStrategy) > 0 {
+		cliConfig.Settings.ListMergeStrategy = configAndStacksInfo.SettingsListMergeStrategy
+		u.LogTrace(*cliConfig, fmt.Sprintf("Using command line argument '%s=%s'", SettingsListMergeStrategyFlag, configAndStacksInfo.SettingsListMergeStrategy))
+	}
 
 	return nil
 }