Add gomplate datasources to Go templates in Atmos stack manifests…

…. Update docs (#582)

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

atmos.yaml

@@ -322,3 +322,6 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      timeout: 5
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

cmd/cmd_utils.go

@@ -219,7 +219,7 @@ func executeCustomCommand(
 		// process the component stack config and expose it in {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables
 		if commandConfig.ComponentConfig.Component != "" && commandConfig.ComponentConfig.Stack != "" {
 			// Process Go templates in the command's 'component_config.component'
-			component, err := u.ProcessTmpl(cliConfig, fmt.Sprintf("component-config-component-%d", i), commandConfig.ComponentConfig.Component, data, false)
+			component, err := u.ProcessTmpl(fmt.Sprintf("component-config-component-%d", i), commandConfig.ComponentConfig.Component, data, false)
 			if err != nil {
 				u.LogErrorAndExit(err)
 			}
@@ -229,7 +229,7 @@ func executeCustomCommand(
 			}
 
 			// Process Go templates in the command's 'component_config.stack'
-			stack, err := u.ProcessTmpl(cliConfig, fmt.Sprintf("component-config-stack-%d", i), commandConfig.ComponentConfig.Stack, data, false)
+			stack, err := u.ProcessTmpl(fmt.Sprintf("component-config-stack-%d", i), commandConfig.ComponentConfig.Stack, data, false)
 			if err != nil {
 				u.LogErrorAndExit(err)
 			}
@@ -271,7 +271,7 @@ func executeCustomCommand(
 				value = strings.TrimRight(res, "\r\n")
 			} else {
 				// Process Go templates in the values of the command's ENV vars
-				value, err = u.ProcessTmpl(cliConfig, fmt.Sprintf("env-var-%d", i), value, data, false)
+				value, err = u.ProcessTmpl(fmt.Sprintf("env-var-%d", i), value, data, false)
 				if err != nil {
 					u.LogErrorAndExit(err)
 				}
@@ -293,7 +293,7 @@ func executeCustomCommand(
 
 		// Process Go templates in the command's steps.
 		// Steps support Go templates and have access to {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables
-		commandToRun, err := u.ProcessTmpl(cliConfig, fmt.Sprintf("step-%d", i), step, data, false)
+		commandToRun, err := u.ProcessTmpl(fmt.Sprintf("step-%d", i), step, data, false)
 		if err != nil {
 			u.LogErrorAndExit(err)
 		}

examples/quick-start/Dockerfile

@@ -6,10 +6,10 @@ ARG GEODESIC_OS=debian
 # https://atmos.tools/
 # https://github.com/cloudposse/atmos
 # https://github.com/cloudposse/atmos/releases
-ARG ATMOS_VERSION=1.68.0
+ARG ATMOS_VERSION=1.70.0
 
 # Terraform: https://github.com/hashicorp/terraform/releases
-ARG TF_VERSION=1.7.5
+ARG TF_VERSION=1.8.0
 
 FROM cloudposse/geodesic:${GEODESIC_VERSION}-${GEODESIC_OS}
 

examples/quick-start/atmos.yaml

@@ -255,3 +255,5 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml

@@ -254,3 +254,5 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

examples/quick-start/stacks/orgs/acme/_defaults.yaml

@@ -4,15 +4,16 @@ vars:
 terraform:
   vars:
     tags:
+      # https://atmos.tools/core-concepts/stacks/templating
       atmos_component: "{{ .atmos_component }}"
       atmos_stack: "{{ .atmos_stack }}"
       atmos_manifest: "{{ .atmos_stack_file }}"
       terraform_workspace: "{{ .workspace }}"
-      # Examples of using the Gomplate and Sprig functions
-      # https://docs.gomplate.ca/functions/strings
-      atmos_component_description: "{{ strings.Title .atmos_component }} component {{ .vars.name | strings.Quote }} provisioned in the stack {{ .atmos_stack | strings.Quote }}"
+      # Examples of using the Sprig and Gomplate functions
       # https://masterminds.github.io/sprig/os.html
       provisioned_by_user: '{{ env "USER" }}'
+      # https://docs.gomplate.ca/functions/strings
+      atmos_component_description: "{{ strings.Title .atmos_component }} component {{ .vars.name | strings.Quote }} provisioned in the stack {{ .atmos_stack | strings.Quote }}"
 
   # Terraform backend configuration
   # https://atmos.tools/core-concepts/components/terraform-backends

examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json

@@ -362,6 +362,9 @@
         },
         "atlantis": {
           "$ref": "#/definitions/atlantis"
+        },
+        "templates": {
+          "$ref": "#/definitions/templates"
         }
       },
       "required": [],
@@ -743,6 +746,12 @@
       "description": "Providers section",
       "additionalProperties": true,
       "title": "providers"
+    },
+    "templates": {
+      "type": "object",
+      "description": "Templates section",
+      "additionalProperties": true,
+      "title": "templates"
     }
   }
 }

examples/tests/atmos.yaml

@@ -380,3 +380,6 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      timeout: 5
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml

@@ -626,3 +626,6 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      timeout: 5
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

examples/tests/stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml

@@ -8,9 +8,6 @@ import:
       region: "{{ .region }}"
       environment: "{{ .environment }}"
 
-  # `Go` templates in the import path
-  - path: "orgs/cp/{{ .tenant }}/{{ .stage }}/_defaults"
-
 components:
   terraform:
     # Parameterize Atmos component name

examples/tests/stacks/orgs/cp/_defaults.yaml

@@ -80,6 +80,17 @@ components:
   helmfile: { }
 
 settings:
+  templates:
+    settings:
+      gomplate:
+        timeout: 20  # 20 seconds timeout to execute the datasources
+        # https://docs.gomplate.ca/datasources
+        datasources:
+          ip:
+            url: "https://api.ipify.org?format=json"
+            headers:
+              accept:
+                - "application/json"
   spacelift:
     workspace_enabled: false
     autodeploy: false

examples/tests/stacks/orgs/cp/tenant1/test1/us-west-1.yaml

@@ -1,4 +1,6 @@
 import:
+  - path: mixins/region/us-west-1
+  - path: orgs/cp/tenant1/test1/_defaults
 
   # This import with the provided hierarchical context will dynamically generate
   # a new Atmos component `eks-blue/cluster` in the `tenant1-uw1-test-1` stack

internal/exec/describe_affected_utils.go

@@ -1122,7 +1122,7 @@ func addAffectedSpaceliftAdminStack(
 	var adminStackContextPrefix string
 
 	if cliConfig.Stacks.NameTemplate != "" {
-		adminStackContextPrefix, err = u.ProcessTmpl(cliConfig, "spacelift-admin-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+		adminStackContextPrefix, err = u.ProcessTmpl("spacelift-admin-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 		if err != nil {
 			return nil, err
 		}
@@ -1158,7 +1158,7 @@ func addAffectedSpaceliftAdminStack(
 							var contextPrefix string
 
 							if cliConfig.Stacks.NameTemplate != "" {
-								contextPrefix, err = u.ProcessTmpl(cliConfig, "spacelift-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+								contextPrefix, err = u.ProcessTmpl("spacelift-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 								if err != nil {
 									return nil, err
 								}

internal/exec/describe_stacks.go

@@ -1,8 +1,10 @@
 package exec
 
 import (
+	"errors"
 	"fmt"
 	c "github.com/cloudposse/atmos/pkg/convert"
+	"github.com/mitchellh/mapstructure"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -208,7 +210,7 @@ func ExecuteDescribeStacks(
 
 						// Stack name
 						if cliConfig.Stacks.NameTemplate != "" {
-							stackName, err = u.ProcessTmpl(cliConfig, "describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+							stackName, err = u.ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 							if err != nil {
 								return nil, err
 							}
@@ -263,16 +265,29 @@ func ExecuteDescribeStacks(
 								return nil, err
 							}
 
-							componentSectionProcessed, err := u.ProcessTmpl(cliConfig, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true)
+							var settingsSectionStruct schema.Settings
+							err = mapstructure.Decode(settingsSection, &settingsSectionStruct)
 							if err != nil {
 								return nil, err
 							}
 
-							componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed)
+							componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true)
 							if err != nil {
 								return nil, err
 							}
 
+							componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed)
+							if err != nil {
+								if !cliConfig.Templates.Settings.Enabled {
+									if strings.Contains(componentSectionStr, "{{") || strings.Contains(componentSectionStr, "}}") {
+										errorMessage := "the stack manifests contain Go templates, but templating is disabled in atmos.yaml in 'templates.settings.enabled'\n" +
+											"to enable templating, refer to https://atmos.tools/core-concepts/stacks/templating"
+										err = errors.Join(err, errors.New(errorMessage))
+									}
+								}
+								u.LogErrorAndExit(err)
+							}
+
 							componentSection = c.MapsOfInterfacesToMapsOfStrings(componentSectionConverted)
 
 							// Add sections
@@ -369,7 +384,7 @@ func ExecuteDescribeStacks(
 
 						// Stack name
 						if cliConfig.Stacks.NameTemplate != "" {
-							stackName, err = u.ProcessTmpl(cliConfig, "describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+							stackName, err = u.ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 							if err != nil {
 								return nil, err
 							}
@@ -416,16 +431,29 @@ func ExecuteDescribeStacks(
 								return nil, err
 							}
 
-							componentSectionProcessed, err := u.ProcessTmpl(cliConfig, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true)
+							var settingsSectionStruct schema.Settings
+							err = mapstructure.Decode(settingsSection, &settingsSectionStruct)
 							if err != nil {
 								return nil, err
 							}
 
-							componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed)
+							componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true)
 							if err != nil {
 								return nil, err
 							}
 
+							componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed)
+							if err != nil {
+								if !cliConfig.Templates.Settings.Enabled {
+									if strings.Contains(componentSectionStr, "{{") || strings.Contains(componentSectionStr, "}}") {
+										errorMessage := "the stack manifests contain Go templates, but templating is disabled in atmos.yaml in 'templates.settings.enabled'\n" +
+											"to enable templating, refer to https://atmos.tools/core-concepts/stacks/templating"
+										err = errors.Join(err, errors.New(errorMessage))
+									}
+								}
+								u.LogErrorAndExit(err)
+							}
+
 							componentSection = c.MapsOfInterfacesToMapsOfStrings(componentSectionConverted)
 
 							// Add sections

internal/exec/spacelift_utils.go

@@ -103,7 +103,7 @@ func BuildSpaceliftStackNameFromComponentConfig(
 		context.Component = strings.Replace(configAndStacksInfo.ComponentFromArg, "/", "-", -1)
 
 		if cliConfig.Stacks.NameTemplate != "" {
-			contextPrefix, err = u.ProcessTmpl(cliConfig, "name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+			contextPrefix, err = u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 			if err != nil {
 				return "", err
 			}

internal/exec/stack_utils.go

@@ -17,7 +17,7 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI
 	var tmpl string
 
 	if cliConfig.Stacks.NameTemplate != "" {
-		tmpl, err = u.ProcessTmpl(cliConfig, "terraform-workspace-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+		tmpl, err = u.ProcessTmpl("terraform-workspace-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 		if err != nil {
 			return "", err
 		}
@@ -36,7 +36,7 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI
 
 	// Terraform workspace can be overridden per component using `metadata.terraform_workspace_pattern` or `metadata.terraform_workspace_template` or `metadata.terraform_workspace`
 	if terraformWorkspaceTemplate, terraformWorkspaceTemplateExist := componentMetadata["terraform_workspace_template"].(string); terraformWorkspaceTemplateExist {
-		tmpl, err = u.ProcessTmpl(cliConfig, "terraform-workspace-template", terraformWorkspaceTemplate, configAndStacksInfo.ComponentSection, false)
+		tmpl, err = u.ProcessTmpl("terraform-workspace-template", terraformWorkspaceTemplate, configAndStacksInfo.ComponentSection, false)
 		if err != nil {
 			return "", err
 		}

internal/exec/utils.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
+	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cobra"
 
 	cfg "github.com/cloudposse/atmos/pkg/config"
@@ -347,7 +348,7 @@ func ProcessStacks(
 			configAndStacksInfo.ComponentEnvList = u.ConvertEnvVars(configAndStacksInfo.ComponentEnvSection)
 
 			if cliConfig.Stacks.NameTemplate != "" {
-				tmpl, err2 := u.ProcessTmpl(cliConfig, "name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+				tmpl, err2 := u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
 				if err2 != nil {
 					continue
 				}
@@ -535,25 +536,40 @@ func ProcessStacks(
 	configAndStacksInfo.ComponentSection["deps"] = componentDeps
 	configAndStacksInfo.ComponentSection["deps_all"] = componentDepsAll
 
-	// Process `Go` templates in sections
+	// Process `Go` templates in Atmos manifest sections
 	componentSectionStr, err := u.ConvertToYAML(configAndStacksInfo.ComponentSection)
 	if err != nil {
 		return configAndStacksInfo, err
 	}
 
-	componentSectionProcessed, err := u.ProcessTmpl(cliConfig, "all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true)
+	var settingsSectionStruct schema.Settings
+
+	err = mapstructure.Decode(configAndStacksInfo.ComponentSettingsSection, &settingsSectionStruct)
+	if err != nil {
+		return configAndStacksInfo, err
+	}
+
+	componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "all-atmos-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true)
 	if err != nil {
 		// If any error returned from the templates processing, log it and exit
 		u.LogErrorAndExit(err)
 	}
 
 	componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed)
 	if err != nil {
-		return configAndStacksInfo, err
+		if !cliConfig.Templates.Settings.Enabled {
+			if strings.Contains(componentSectionStr, "{{") || strings.Contains(componentSectionStr, "}}") {
+				errorMessage := "the stack manifests contain Go templates, but templating is disabled in atmos.yaml in 'templates.settings.enabled'\n" +
+					"to enable templating, refer to https://atmos.tools/core-concepts/stacks/templating"
+				err = errors.Join(err, errors.New(errorMessage))
+			}
+		}
+		u.LogErrorAndExit(err)
 	}
 
 	configAndStacksInfo.ComponentSection = c.MapsOfInterfacesToMapsOfStrings(componentSectionConverted)
 
+	// Process Atmos manifest sections
 	if i, ok := configAndStacksInfo.ComponentSection[cfg.ProvidersSectionName].(map[any]any); ok {
 		configAndStacksInfo.ComponentProvidersSection = i
 	}

internal/exec/vendor_utils.go

@@ -284,7 +284,7 @@ func ExecuteAtmosVendorInternal(
 
 		// Parse 'source' template
 		if s.Version != "" {
-			uri, err = u.ProcessTmpl(cliConfig, fmt.Sprintf("source-%d-%s", indexSource, s.Version), s.Source, s, false)
+			uri, err = u.ProcessTmpl(fmt.Sprintf("source-%d-%s", indexSource, s.Version), s.Source, s, false)
 			if err != nil {
 				return err
 			}
@@ -318,7 +318,7 @@ func ExecuteAtmosVendorInternal(
 			var target string
 			// Parse 'target' template
 			if s.Version != "" {
-				target, err = u.ProcessTmpl(cliConfig, fmt.Sprintf("target-%d-%d-%s", indexSource, indexTarget, s.Version), tgt, s, false)
+				target, err = u.ProcessTmpl(fmt.Sprintf("target-%d-%d-%s", indexSource, indexTarget, s.Version), tgt, s, false)
 				if err != nil {
 					return err
 				}

pkg/atlantis/atmos.yaml

@@ -358,3 +358,6 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      timeout: 5
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

pkg/aws/atmos.yaml

@@ -358,3 +358,6 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      timeout: 5
+      # https://docs.gomplate.ca/datasources
+      datasources: {}

pkg/component/atmos.yaml

@@ -358,3 +358,6 @@ templates:
     # https://docs.gomplate.ca
     gomplate:
       enabled: true
+      timeout: 5
+      # https://docs.gomplate.ca/datasources
+      datasources: {}