diff --git a/cmd/cmd_utils.go b/cmd/cmd_utils.go index 8f70fe126..e59ef46d5 100644 --- a/cmd/cmd_utils.go +++ b/cmd/cmd_utils.go @@ -219,7 +219,7 @@ func executeCustomCommand( // process the component stack config and expose it in {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables if commandConfig.ComponentConfig.Component != "" && commandConfig.ComponentConfig.Stack != "" { // Process Go templates in the command's 'component_config.component' - component, err := u.ProcessTmpl(fmt.Sprintf("component-config-component-%d", i), commandConfig.ComponentConfig.Component, data, false) + component, err := e.ProcessTmpl(fmt.Sprintf("component-config-component-%d", i), commandConfig.ComponentConfig.Component, data, false) if err != nil { u.LogErrorAndExit(err) } @@ -229,7 +229,7 @@ func executeCustomCommand( } // Process Go templates in the command's 'component_config.stack' - stack, err := u.ProcessTmpl(fmt.Sprintf("component-config-stack-%d", i), commandConfig.ComponentConfig.Stack, data, false) + stack, err := e.ProcessTmpl(fmt.Sprintf("component-config-stack-%d", i), commandConfig.ComponentConfig.Stack, data, false) if err != nil { u.LogErrorAndExit(err) } @@ -271,7 +271,7 @@ func executeCustomCommand( value = strings.TrimRight(res, "\r\n") } else { // Process Go templates in the values of the command's ENV vars - value, err = u.ProcessTmpl(fmt.Sprintf("env-var-%d", i), value, data, false) + value, err = e.ProcessTmpl(fmt.Sprintf("env-var-%d", i), value, data, false) if err != nil { u.LogErrorAndExit(err) } @@ -293,7 +293,7 @@ func executeCustomCommand( // Process Go templates in the command's steps. // Steps support Go templates and have access to {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables - commandToRun, err := u.ProcessTmpl(fmt.Sprintf("step-%d", i), step, data, false) + commandToRun, err := e.ProcessTmpl(fmt.Sprintf("step-%d", i), step, data, false) if err != nil { u.LogErrorAndExit(err) } diff --git a/examples/quick-start/Dockerfile b/examples/quick-start/Dockerfile index 15c9da304..fa929818e 100644 --- a/examples/quick-start/Dockerfile +++ b/examples/quick-start/Dockerfile @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.80.0 +ARG ATMOS_VERSION=1.81.0 # Terraform: https://github.com/hashicorp/terraform/releases ARG TF_VERSION=1.8.5 diff --git a/examples/tests/components/terraform/test/template-functions-test/context.tf b/examples/tests/components/terraform/test/template-functions-test/context.tf new file mode 100644 index 000000000..5e0ef8856 --- /dev/null +++ b/examples/tests/components/terraform/test/template-functions-test/context.tf @@ -0,0 +1,279 @@ +# +# ONLY EDIT THIS FILE IN github.com/cloudposse/terraform-null-label +# All other instances of this file should be a copy of that one +# +# +# Copy this file from https://github.com/cloudposse/terraform-null-label/blob/master/exports/context.tf +# and then place it in your Terraform module to automatically get +# Cloud Posse's standard configuration inputs suitable for passing +# to Cloud Posse modules. +# +# curl -sL https://raw.githubusercontent.com/cloudposse/terraform-null-label/master/exports/context.tf -o context.tf +# +# Modules should access the whole context as `module.this.context` +# to get the input variables with nulls for defaults, +# for example `context = module.this.context`, +# and access individual variables as `module.this.`, +# with final values filled in. +# +# For example, when using defaults, `module.this.context.delimiter` +# will be null, and `module.this.delimiter` will be `-` (hyphen). +# + +module "this" { + source = "cloudposse/label/null" + version = "0.25.0" # requires Terraform >= 0.13.0 + + enabled = var.enabled + namespace = var.namespace + tenant = var.tenant + environment = var.environment + stage = var.stage + name = var.name + delimiter = var.delimiter + attributes = var.attributes + tags = var.tags + additional_tag_map = var.additional_tag_map + label_order = var.label_order + regex_replace_chars = var.regex_replace_chars + id_length_limit = var.id_length_limit + label_key_case = var.label_key_case + label_value_case = var.label_value_case + descriptor_formats = var.descriptor_formats + labels_as_tags = var.labels_as_tags + + context = var.context +} + +# Copy contents of cloudposse/terraform-null-label/variables.tf here + +variable "context" { + type = any + default = { + enabled = true + namespace = null + tenant = null + environment = null + stage = null + name = null + delimiter = null + attributes = [] + tags = {} + additional_tag_map = {} + regex_replace_chars = null + label_order = [] + id_length_limit = null + label_key_case = null + label_value_case = null + descriptor_formats = {} + # Note: we have to use [] instead of null for unset lists due to + # https://github.com/hashicorp/terraform/issues/28137 + # which was not fixed until Terraform 1.0.0, + # but we want the default to be all the labels in `label_order` + # and we want users to be able to prevent all tag generation + # by setting `labels_as_tags` to `[]`, so we need + # a different sentinel to indicate "default" + labels_as_tags = ["unset"] + } + description = <<-EOT + Single object for setting entire context at once. + See description of individual variables for details. + Leave string and numeric variables as `null` to use default value. + Individual variable settings (non-null) override settings in context object, + except for attributes, tags, and additional_tag_map, which are merged. + EOT + + validation { + condition = lookup(var.context, "label_key_case", null) == null ? true : contains(["lower", "title", "upper"], var.context["label_key_case"]) + error_message = "Allowed values: `lower`, `title`, `upper`." + } + + validation { + condition = lookup(var.context, "label_value_case", null) == null ? true : contains(["lower", "title", "upper", "none"], var.context["label_value_case"]) + error_message = "Allowed values: `lower`, `title`, `upper`, `none`." + } +} + +variable "enabled" { + type = bool + default = null + description = "Set to false to prevent the module from creating any resources" +} + +variable "namespace" { + type = string + default = null + description = "ID element. Usually an abbreviation of your organization name, e.g. 'eg' or 'cp', to help ensure generated IDs are globally unique" +} + +variable "tenant" { + type = string + default = null + description = "ID element _(Rarely used, not included by default)_. A customer identifier, indicating who this instance of a resource is for" +} + +variable "environment" { + type = string + default = null + description = "ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT'" +} + +variable "stage" { + type = string + default = null + description = "ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release'" +} + +variable "name" { + type = string + default = null + description = <<-EOT + ID element. Usually the component or solution name, e.g. 'app' or 'jenkins'. + This is the only ID element not also included as a `tag`. + The "name" tag is set to the full `id` string. There is no tag with the value of the `name` input. + EOT +} + +variable "delimiter" { + type = string + default = null + description = <<-EOT + Delimiter to be used between ID elements. + Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. + EOT +} + +variable "attributes" { + type = list(string) + default = [] + description = <<-EOT + ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`, + in the order they appear in the list. New attributes are appended to the + end of the list. The elements of the list are joined by the `delimiter` + and treated as a single ID element. + EOT +} + +variable "labels_as_tags" { + type = set(string) + default = ["default"] + description = <<-EOT + Set of labels (ID elements) to include as tags in the `tags` output. + Default is to include all labels. + Tags with empty values will not be included in the `tags` output. + Set to `[]` to suppress all generated tags. + **Notes:** + The value of the `name` tag, if included, will be the `id`, not the `name`. + Unlike other `null-label` inputs, the initial setting of `labels_as_tags` cannot be + changed in later chained modules. Attempts to change it will be silently ignored. + EOT +} + +variable "tags" { + type = map(string) + default = {} + description = <<-EOT + Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`). + Neither the tag keys nor the tag values will be modified by this module. + EOT +} + +variable "additional_tag_map" { + type = map(string) + default = {} + description = <<-EOT + Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`. + This is for some rare cases where resources want additional configuration of tags + and therefore take a list of maps with tag key, value, and additional configuration. + EOT +} + +variable "label_order" { + type = list(string) + default = null + description = <<-EOT + The order in which the labels (ID elements) appear in the `id`. + Defaults to ["namespace", "environment", "stage", "name", "attributes"]. + You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. + EOT +} + +variable "regex_replace_chars" { + type = string + default = null + description = <<-EOT + Terraform regular expression (regex) string. + Characters matching the regex will be removed from the ID elements. + If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. + EOT +} + +variable "id_length_limit" { + type = number + default = null + description = <<-EOT + Limit `id` to this many characters (minimum 6). + Set to `0` for unlimited length. + Set to `null` for keep the existing setting, which defaults to `0`. + Does not affect `id_full`. + EOT + validation { + condition = var.id_length_limit == null ? true : var.id_length_limit >= 6 || var.id_length_limit == 0 + error_message = "The id_length_limit must be >= 6 if supplied (not null), or 0 for unlimited length." + } +} + +variable "label_key_case" { + type = string + default = null + description = <<-EOT + Controls the letter case of the `tags` keys (label names) for tags generated by this module. + Does not affect keys of tags passed in via the `tags` input. + Possible values: `lower`, `title`, `upper`. + Default value: `title`. + EOT + + validation { + condition = var.label_key_case == null ? true : contains(["lower", "title", "upper"], var.label_key_case) + error_message = "Allowed values: `lower`, `title`, `upper`." + } +} + +variable "label_value_case" { + type = string + default = null + description = <<-EOT + Controls the letter case of ID elements (labels) as included in `id`, + set as tag values, and output by this module individually. + Does not affect values of tags passed in via the `tags` input. + Possible values: `lower`, `title`, `upper` and `none` (no transformation). + Set this to `title` and set `delimiter` to `""` to yield Pascal Case IDs. + Default value: `lower`. + EOT + + validation { + condition = var.label_value_case == null ? true : contains(["lower", "title", "upper", "none"], var.label_value_case) + error_message = "Allowed values: `lower`, `title`, `upper`, `none`." + } +} + +variable "descriptor_formats" { + type = any + default = {} + description = <<-EOT + Describe additional descriptors to be output in the `descriptors` output map. + Map of maps. Keys are names of descriptors. Values are maps of the form + `{ + format = string + labels = list(string) + }` + (Type is `any` so the map values can later be enhanced to provide additional options.) + `format` is a Terraform format string to be passed to the `format()` function. + `labels` is a list of labels, in order, to pass to `format()` function. + Label values will be normalized before being passed to `format()` so they will be + identical to how they appear in `id`. + Default is `{}` (`descriptors` output will be empty). + EOT +} + +#### End of copy of cloudposse/terraform-null-label/variables.tf diff --git a/examples/tests/components/terraform/test/template-functions-test/main.tf b/examples/tests/components/terraform/test/template-functions-test/main.tf new file mode 100644 index 000000000..3d0c15f53 --- /dev/null +++ b/examples/tests/components/terraform/test/template-functions-test/main.tf @@ -0,0 +1,6 @@ +module "test_label" { + source = "cloudposse/label/null" + version = "0.25.0" + + context = module.this.context +} diff --git a/examples/tests/components/terraform/test/template-functions-test/outputs.tf b/examples/tests/components/terraform/test/template-functions-test/outputs.tf new file mode 100644 index 000000000..72cf38561 --- /dev/null +++ b/examples/tests/components/terraform/test/template-functions-test/outputs.tf @@ -0,0 +1,4 @@ +output "test_label_id" { + value = module.test_label.id + description = "Test label ID" +} diff --git a/examples/tests/components/terraform/test/template-functions-test2/context.tf b/examples/tests/components/terraform/test/template-functions-test2/context.tf new file mode 100644 index 000000000..5e0ef8856 --- /dev/null +++ b/examples/tests/components/terraform/test/template-functions-test2/context.tf @@ -0,0 +1,279 @@ +# +# ONLY EDIT THIS FILE IN github.com/cloudposse/terraform-null-label +# All other instances of this file should be a copy of that one +# +# +# Copy this file from https://github.com/cloudposse/terraform-null-label/blob/master/exports/context.tf +# and then place it in your Terraform module to automatically get +# Cloud Posse's standard configuration inputs suitable for passing +# to Cloud Posse modules. +# +# curl -sL https://raw.githubusercontent.com/cloudposse/terraform-null-label/master/exports/context.tf -o context.tf +# +# Modules should access the whole context as `module.this.context` +# to get the input variables with nulls for defaults, +# for example `context = module.this.context`, +# and access individual variables as `module.this.`, +# with final values filled in. +# +# For example, when using defaults, `module.this.context.delimiter` +# will be null, and `module.this.delimiter` will be `-` (hyphen). +# + +module "this" { + source = "cloudposse/label/null" + version = "0.25.0" # requires Terraform >= 0.13.0 + + enabled = var.enabled + namespace = var.namespace + tenant = var.tenant + environment = var.environment + stage = var.stage + name = var.name + delimiter = var.delimiter + attributes = var.attributes + tags = var.tags + additional_tag_map = var.additional_tag_map + label_order = var.label_order + regex_replace_chars = var.regex_replace_chars + id_length_limit = var.id_length_limit + label_key_case = var.label_key_case + label_value_case = var.label_value_case + descriptor_formats = var.descriptor_formats + labels_as_tags = var.labels_as_tags + + context = var.context +} + +# Copy contents of cloudposse/terraform-null-label/variables.tf here + +variable "context" { + type = any + default = { + enabled = true + namespace = null + tenant = null + environment = null + stage = null + name = null + delimiter = null + attributes = [] + tags = {} + additional_tag_map = {} + regex_replace_chars = null + label_order = [] + id_length_limit = null + label_key_case = null + label_value_case = null + descriptor_formats = {} + # Note: we have to use [] instead of null for unset lists due to + # https://github.com/hashicorp/terraform/issues/28137 + # which was not fixed until Terraform 1.0.0, + # but we want the default to be all the labels in `label_order` + # and we want users to be able to prevent all tag generation + # by setting `labels_as_tags` to `[]`, so we need + # a different sentinel to indicate "default" + labels_as_tags = ["unset"] + } + description = <<-EOT + Single object for setting entire context at once. + See description of individual variables for details. + Leave string and numeric variables as `null` to use default value. + Individual variable settings (non-null) override settings in context object, + except for attributes, tags, and additional_tag_map, which are merged. + EOT + + validation { + condition = lookup(var.context, "label_key_case", null) == null ? true : contains(["lower", "title", "upper"], var.context["label_key_case"]) + error_message = "Allowed values: `lower`, `title`, `upper`." + } + + validation { + condition = lookup(var.context, "label_value_case", null) == null ? true : contains(["lower", "title", "upper", "none"], var.context["label_value_case"]) + error_message = "Allowed values: `lower`, `title`, `upper`, `none`." + } +} + +variable "enabled" { + type = bool + default = null + description = "Set to false to prevent the module from creating any resources" +} + +variable "namespace" { + type = string + default = null + description = "ID element. Usually an abbreviation of your organization name, e.g. 'eg' or 'cp', to help ensure generated IDs are globally unique" +} + +variable "tenant" { + type = string + default = null + description = "ID element _(Rarely used, not included by default)_. A customer identifier, indicating who this instance of a resource is for" +} + +variable "environment" { + type = string + default = null + description = "ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT'" +} + +variable "stage" { + type = string + default = null + description = "ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release'" +} + +variable "name" { + type = string + default = null + description = <<-EOT + ID element. Usually the component or solution name, e.g. 'app' or 'jenkins'. + This is the only ID element not also included as a `tag`. + The "name" tag is set to the full `id` string. There is no tag with the value of the `name` input. + EOT +} + +variable "delimiter" { + type = string + default = null + description = <<-EOT + Delimiter to be used between ID elements. + Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. + EOT +} + +variable "attributes" { + type = list(string) + default = [] + description = <<-EOT + ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`, + in the order they appear in the list. New attributes are appended to the + end of the list. The elements of the list are joined by the `delimiter` + and treated as a single ID element. + EOT +} + +variable "labels_as_tags" { + type = set(string) + default = ["default"] + description = <<-EOT + Set of labels (ID elements) to include as tags in the `tags` output. + Default is to include all labels. + Tags with empty values will not be included in the `tags` output. + Set to `[]` to suppress all generated tags. + **Notes:** + The value of the `name` tag, if included, will be the `id`, not the `name`. + Unlike other `null-label` inputs, the initial setting of `labels_as_tags` cannot be + changed in later chained modules. Attempts to change it will be silently ignored. + EOT +} + +variable "tags" { + type = map(string) + default = {} + description = <<-EOT + Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`). + Neither the tag keys nor the tag values will be modified by this module. + EOT +} + +variable "additional_tag_map" { + type = map(string) + default = {} + description = <<-EOT + Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`. + This is for some rare cases where resources want additional configuration of tags + and therefore take a list of maps with tag key, value, and additional configuration. + EOT +} + +variable "label_order" { + type = list(string) + default = null + description = <<-EOT + The order in which the labels (ID elements) appear in the `id`. + Defaults to ["namespace", "environment", "stage", "name", "attributes"]. + You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. + EOT +} + +variable "regex_replace_chars" { + type = string + default = null + description = <<-EOT + Terraform regular expression (regex) string. + Characters matching the regex will be removed from the ID elements. + If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. + EOT +} + +variable "id_length_limit" { + type = number + default = null + description = <<-EOT + Limit `id` to this many characters (minimum 6). + Set to `0` for unlimited length. + Set to `null` for keep the existing setting, which defaults to `0`. + Does not affect `id_full`. + EOT + validation { + condition = var.id_length_limit == null ? true : var.id_length_limit >= 6 || var.id_length_limit == 0 + error_message = "The id_length_limit must be >= 6 if supplied (not null), or 0 for unlimited length." + } +} + +variable "label_key_case" { + type = string + default = null + description = <<-EOT + Controls the letter case of the `tags` keys (label names) for tags generated by this module. + Does not affect keys of tags passed in via the `tags` input. + Possible values: `lower`, `title`, `upper`. + Default value: `title`. + EOT + + validation { + condition = var.label_key_case == null ? true : contains(["lower", "title", "upper"], var.label_key_case) + error_message = "Allowed values: `lower`, `title`, `upper`." + } +} + +variable "label_value_case" { + type = string + default = null + description = <<-EOT + Controls the letter case of ID elements (labels) as included in `id`, + set as tag values, and output by this module individually. + Does not affect values of tags passed in via the `tags` input. + Possible values: `lower`, `title`, `upper` and `none` (no transformation). + Set this to `title` and set `delimiter` to `""` to yield Pascal Case IDs. + Default value: `lower`. + EOT + + validation { + condition = var.label_value_case == null ? true : contains(["lower", "title", "upper", "none"], var.label_value_case) + error_message = "Allowed values: `lower`, `title`, `upper`, `none`." + } +} + +variable "descriptor_formats" { + type = any + default = {} + description = <<-EOT + Describe additional descriptors to be output in the `descriptors` output map. + Map of maps. Keys are names of descriptors. Values are maps of the form + `{ + format = string + labels = list(string) + }` + (Type is `any` so the map values can later be enhanced to provide additional options.) + `format` is a Terraform format string to be passed to the `format()` function. + `labels` is a list of labels, in order, to pass to `format()` function. + Label values will be normalized before being passed to `format()` so they will be + identical to how they appear in `id`. + Default is `{}` (`descriptors` output will be empty). + EOT +} + +#### End of copy of cloudposse/terraform-null-label/variables.tf diff --git a/examples/tests/components/terraform/test/template-functions-test2/outputs.tf b/examples/tests/components/terraform/test/template-functions-test2/outputs.tf new file mode 100644 index 000000000..953aac8e7 --- /dev/null +++ b/examples/tests/components/terraform/test/template-functions-test2/outputs.tf @@ -0,0 +1,9 @@ +output "test_label_id" { + value = var.test_label_id + description = "Test label ID" +} + +output "region" { + value = var.region + description = "Test label ID" +} diff --git a/examples/tests/components/terraform/test/template-functions-test2/variables.tf b/examples/tests/components/terraform/test/template-functions-test2/variables.tf new file mode 100644 index 000000000..c4e420970 --- /dev/null +++ b/examples/tests/components/terraform/test/template-functions-test2/variables.tf @@ -0,0 +1,9 @@ +variable "test_label_id" { + type = string + description = "Test label ID" +} + +variable "region" { + type = string + description = "Region" +} diff --git a/examples/tests/stacks/catalog/terraform/template-functions-test/defaults.yaml b/examples/tests/stacks/catalog/terraform/template-functions-test/defaults.yaml new file mode 100644 index 000000000..6a6d3b5f3 --- /dev/null +++ b/examples/tests/stacks/catalog/terraform/template-functions-test/defaults.yaml @@ -0,0 +1,10 @@ +components: + terraform: + template-functions-test: + command: tofu + metadata: + # Point to the Terraform component + component: "test/template-functions-test" + vars: + enabled: true + name: "test" diff --git a/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml b/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml new file mode 100644 index 000000000..fac8fbc6d --- /dev/null +++ b/examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml @@ -0,0 +1,11 @@ +components: + terraform: + template-functions-test2: + metadata: + # Point to the Terraform component + component: "test/template-functions-test2" + vars: + enabled: true + name: "test2" + # Examples of using Atmos template functions to get the outputs of another Atmos component + test_label_id: '{{ (atmos.Component "template-functions-test" .stack).outputs.test_label_id }}' diff --git a/go.mod b/go.mod index ae7ffc487..dd1214998 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/elewis787/boa v0.1.2 github.com/fatih/color v1.17.0 github.com/go-git/go-git/v5 v5.12.0 - github.com/google/go-containerregistry v0.19.1 + github.com/google/go-containerregistry v0.19.2 github.com/google/go-github/v59 v59.0.0 github.com/google/uuid v1.6.0 github.com/hairyhenderson/gomplate/v3 v3.11.8 @@ -22,10 +22,11 @@ require ( github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl/v2 v2.20.1 github.com/hashicorp/terraform-config-inspect v0.0.0-20240607080351-271db412dbcb + github.com/hashicorp/terraform-exec v0.21.0 github.com/ivanpirog/coloredcobra v1.0.1 github.com/json-iterator/go v1.1.12 github.com/jwalton/go-supportscolor v1.2.0 - github.com/lrstanley/bubblezone v0.0.0-20240609171605-b723e5c0b1af + github.com/lrstanley/bubblezone v0.0.0-20240615043033-81c0dd750246 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/open-policy-agent/opa v0.65.0 @@ -33,7 +34,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/samber/lo v1.39.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 @@ -54,7 +55,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.11.5 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect - github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect github.com/Shopify/ejson v1.3.3 // indirect github.com/agext/levenshtein v1.2.2 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect @@ -150,6 +151,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/serf v0.10.1 // indirect + github.com/hashicorp/terraform-json v0.22.1 // indirect github.com/hashicorp/vault/api v1.6.0 // indirect github.com/hashicorp/vault/sdk v0.5.0 // indirect github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect diff --git a/go.sum b/go.sum index 0bd2ca92c..eb1088c4e 100644 --- a/go.sum +++ b/go.sum @@ -257,8 +257,8 @@ github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= -github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg= +github.com/ProtonMail/go-crypto v1.1.0-alpha.2/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/Shopify/ejson v1.3.3 h1:dPzgmvFhUPTJIzwdF5DaqbwW1dWaoR8ADKRdSTy6Mss= github.com/Shopify/ejson v1.3.3/go.mod h1:VZMUtDzvBW/PAXRUF5fzp1ffb1ucT8MztrZXXLYZurw= github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= @@ -369,7 +369,6 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= @@ -405,7 +404,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -435,7 +433,7 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= @@ -633,8 +631,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= -github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w= +github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/go-github/v59 v59.0.0 h1:7h6bgpF5as0YQLLkEiVqpgtJqjimMYhBkD4jT5aN3VA= github.com/google/go-github/v59 v59.0.0/go.mod h1:rJU4R0rQHFVFDOkqGWxfLNo6vEk4dv40oDjhV/gH6wM= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -787,6 +785,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hc-install v0.6.4 h1:QLqlM56/+SIIGvGcfFiwMY3z5WGXT066suo/v9Km8e0= +github.com/hashicorp/hc-install v0.6.4/go.mod h1:05LWLy8TD842OtgcfBbOT0WMoInBMUSHjmDx10zuBIA= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= @@ -799,6 +799,10 @@ github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/terraform-config-inspect v0.0.0-20240607080351-271db412dbcb h1:6gCfY5aQdQgRr0G5VDjnV5ENpd+hTamWaZfVz+lJ724= github.com/hashicorp/terraform-config-inspect v0.0.0-20240607080351-271db412dbcb/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= +github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= +github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= +github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= +github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= github.com/hashicorp/vault/api v1.6.0 h1:B8UUYod1y1OoiGHq9GtpiqSnGOUEWHaA26AY8RQEDY4= github.com/hashicorp/vault/api v1.6.0/go.mod h1:h1K70EO2DgnBaTz5IsL6D5ERsNt5Pce93ueVS2+t0Xc= github.com/hashicorp/vault/sdk v0.5.0 h1:EED7p0OCU3OY5SAqJwSANofY1YKMytm+jDHDQ2EzGVQ= @@ -912,8 +916,8 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lrstanley/bubblezone v0.0.0-20240609171605-b723e5c0b1af h1:ElLxMeBKoghsTLsrDyZGLcEz6WcPZcBzYvAK9CIHDyA= -github.com/lrstanley/bubblezone v0.0.0-20240609171605-b723e5c0b1af/go.mod h1:k7GwzSav+E8Isx3+2d5z5EJhIQrbC7mIOAFvV0TylSI= +github.com/lrstanley/bubblezone v0.0.0-20240615043033-81c0dd750246 h1:s5kInFIBoY0KCSi49AqHIUvkGF/hGfKetdOBN41UWCg= +github.com/lrstanley/bubblezone v0.0.0-20240615043033-81c0dd750246/go.mod h1:k7GwzSav+E8Isx3+2d5z5EJhIQrbC7mIOAFvV0TylSI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -1116,8 +1120,8 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= @@ -1262,8 +1266,6 @@ golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1305,7 +1307,6 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1368,8 +1369,6 @@ golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1415,7 +1414,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1510,8 +1508,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1521,8 +1517,6 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1536,8 +1530,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1608,7 +1600,6 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index 4feb3ce16..7b2fe6a42 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -1346,7 +1346,7 @@ func addAffectedSpaceliftAdminStack( var adminStackContextPrefix string if cliConfig.Stacks.NameTemplate != "" { - adminStackContextPrefix, err = u.ProcessTmpl("spacelift-admin-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + adminStackContextPrefix, err = ProcessTmpl("spacelift-admin-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -1382,7 +1382,7 @@ func addAffectedSpaceliftAdminStack( var contextPrefix string if cliConfig.Stacks.NameTemplate != "" { - contextPrefix, err = u.ProcessTmpl("spacelift-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + contextPrefix, err = ProcessTmpl("spacelift-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } diff --git a/internal/exec/describe_stacks.go b/internal/exec/describe_stacks.go index a1549a2e4..ed1ac5418 100644 --- a/internal/exec/describe_stacks.go +++ b/internal/exec/describe_stacks.go @@ -11,7 +11,6 @@ import ( cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" - s "github.com/cloudposse/atmos/pkg/stack" u "github.com/cloudposse/atmos/pkg/utils" ) @@ -145,7 +144,7 @@ func ExecuteDescribeStacks( } // Find all derived components of the provided components and include them in the output - derivedComponents, err := s.FindComponentsDerivedFromBaseComponents(stackFileName, terraformSection, components) + derivedComponents, err := FindComponentsDerivedFromBaseComponents(stackFileName, terraformSection, components) if err != nil { return nil, err } @@ -211,7 +210,7 @@ func ExecuteDescribeStacks( // Stack name if cliConfig.Stacks.NameTemplate != "" { - stackName, err = u.ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + stackName, err = ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -238,7 +237,9 @@ func ExecuteDescribeStacks( configAndStacksInfo.ComponentSection["atmos_component"] = componentName configAndStacksInfo.ComponentSection["atmos_stack"] = stackName + configAndStacksInfo.ComponentSection["stack"] = stackName configAndStacksInfo.ComponentSection["atmos_stack_file"] = stackFileName + configAndStacksInfo.ComponentSection["atmos_manifest"] = stackFileName if len(components) == 0 || u.SliceContainsString(components, componentName) || u.SliceContainsString(derivedComponents, componentName) { if !u.MapKeyExists(finalStacksMap[stackName].(map[string]any), "components") { @@ -262,7 +263,9 @@ func ExecuteDescribeStacks( // Atmos component, stack, and stack manifest file componentSection["atmos_component"] = componentName componentSection["atmos_stack"] = stackName + componentSection["stack"] = stackName componentSection["atmos_stack_file"] = stackFileName + componentSection["atmos_manifest"] = stackFileName // Process `Go` templates componentSectionStr, err := u.ConvertToYAML(componentSection) @@ -276,7 +279,14 @@ func ExecuteDescribeStacks( return nil, err } - componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) + componentSectionProcessed, err := ProcessTmplWithDatasources( + cliConfig, + settingsSectionStruct, + "describe-stacks-all-sections", + componentSectionStr, + configAndStacksInfo.ComponentSection, + true, + ) if err != nil { return nil, err } @@ -319,7 +329,7 @@ func ExecuteDescribeStacks( } // Find all derived components of the provided components and include them in the output - derivedComponents, err := s.FindComponentsDerivedFromBaseComponents(stackFileName, helmfileSection, components) + derivedComponents, err := FindComponentsDerivedFromBaseComponents(stackFileName, helmfileSection, components) if err != nil { return nil, err } @@ -385,7 +395,7 @@ func ExecuteDescribeStacks( // Stack name if cliConfig.Stacks.NameTemplate != "" { - stackName, err = u.ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + stackName, err = ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -412,7 +422,9 @@ func ExecuteDescribeStacks( configAndStacksInfo.ComponentSection["atmos_component"] = componentName configAndStacksInfo.ComponentSection["atmos_stack"] = stackName + configAndStacksInfo.ComponentSection["stack"] = stackName configAndStacksInfo.ComponentSection["atmos_stack_file"] = stackFileName + configAndStacksInfo.ComponentSection["atmos_manifest"] = stackFileName if len(components) == 0 || u.SliceContainsString(components, componentName) || u.SliceContainsString(derivedComponents, componentName) { if !u.MapKeyExists(finalStacksMap[stackName].(map[string]any), "components") { @@ -428,7 +440,9 @@ func ExecuteDescribeStacks( // Atmos component, stack, and stack manifest file componentSection["atmos_component"] = componentName componentSection["atmos_stack"] = stackName + componentSection["stack"] = stackName componentSection["atmos_stack_file"] = stackFileName + componentSection["atmos_manifest"] = stackFileName // Process `Go` templates componentSectionStr, err := u.ConvertToYAML(componentSection) @@ -442,7 +456,14 @@ func ExecuteDescribeStacks( return nil, err } - componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) + componentSectionProcessed, err := ProcessTmplWithDatasources( + cliConfig, + settingsSectionStruct, + "describe-stacks-all-sections", + componentSectionStr, + configAndStacksInfo.ComponentSection, + true, + ) if err != nil { return nil, err } diff --git a/internal/exec/spacelift_utils.go b/internal/exec/spacelift_utils.go index 6598ed834..18a911f3a 100644 --- a/internal/exec/spacelift_utils.go +++ b/internal/exec/spacelift_utils.go @@ -6,7 +6,6 @@ import ( cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" - u "github.com/cloudposse/atmos/pkg/utils" ) // BuildSpaceliftStackName builds a Spacelift stack name from the provided context and stack name pattern @@ -103,7 +102,7 @@ func BuildSpaceliftStackNameFromComponentConfig( context.Component = strings.Replace(configAndStacksInfo.ComponentFromArg, "/", "-", -1) if cliConfig.Stacks.NameTemplate != "" { - contextPrefix, err = u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + contextPrefix, err = ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return "", err } diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go new file mode 100644 index 000000000..2bef13f0c --- /dev/null +++ b/internal/exec/stack_processor_utils.go @@ -0,0 +1,2124 @@ +package exec + +import ( + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/santhosh-tekuri/jsonschema/v5" + "gopkg.in/yaml.v2" + + cfg "github.com/cloudposse/atmos/pkg/config" + c "github.com/cloudposse/atmos/pkg/convert" + m "github.com/cloudposse/atmos/pkg/merge" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" +) + +var ( + getFileContentSyncMap = sync.Map{} + + // Mutex to serialize updates of the result map of ProcessYAMLConfigFiles function + processYAMLConfigFilesLock = &sync.Mutex{} +) + +// ProcessYAMLConfigFiles takes a list of paths to stack manifests, processes and deep-merges all imports, +// and returns a list of stack configs +func ProcessYAMLConfigFiles( + cliConfig schema.CliConfiguration, + stacksBasePath string, + terraformComponentsBasePath string, + helmfileComponentsBasePath string, + filePaths []string, + processStackDeps bool, + processComponentDeps bool, + ignoreMissingFiles bool, +) ( + []string, + map[string]any, + map[string]map[string]any, + error, +) { + + count := len(filePaths) + listResult := make([]string, count) + mapResult := map[string]any{} + rawStackConfigs := map[string]map[string]any{} + var errorResult error + var wg sync.WaitGroup + wg.Add(count) + + for i, filePath := range filePaths { + go func(i int, p string) { + defer wg.Done() + + stackBasePath := stacksBasePath + if len(stackBasePath) < 1 { + stackBasePath = path.Dir(p) + } + + stackFileName := strings.TrimSuffix( + strings.TrimSuffix( + u.TrimBasePathFromPath(stackBasePath+"/", p), + cfg.DefaultStackConfigFileExtension), + ".yml", + ) + + deepMergedStackConfig, importsConfig, stackConfig, err := ProcessYAMLConfigFile( + cliConfig, + stackBasePath, + p, + map[string]map[any]any{}, + nil, + ignoreMissingFiles, + false, + false, + false, + map[any]any{}, + map[any]any{}, + "", + ) + + if err != nil { + errorResult = err + return + } + + var imports []string + for k := range importsConfig { + imports = append(imports, k) + } + + uniqueImports := u.UniqueStrings(imports) + sort.Strings(uniqueImports) + + componentStackMap := map[string]map[string][]string{} + + finalConfig, err := ProcessStackConfig( + cliConfig, + stackBasePath, + terraformComponentsBasePath, + helmfileComponentsBasePath, + p, + deepMergedStackConfig, + processStackDeps, + processComponentDeps, + "", + componentStackMap, + importsConfig, + true) + if err != nil { + errorResult = err + return + } + + finalConfig["imports"] = uniqueImports + + yamlConfig, err := yaml.Marshal(finalConfig) + if err != nil { + errorResult = err + return + } + + processYAMLConfigFilesLock.Lock() + defer processYAMLConfigFilesLock.Unlock() + + listResult[i] = string(yamlConfig) + mapResult[stackFileName] = finalConfig + rawStackConfigs[stackFileName] = map[string]any{} + rawStackConfigs[stackFileName]["stack"] = stackConfig + rawStackConfigs[stackFileName]["imports"] = importsConfig + rawStackConfigs[stackFileName]["import_files"] = uniqueImports + }(i, filePath) + } + + wg.Wait() + + if errorResult != nil { + return nil, nil, nil, errorResult + } + + return listResult, mapResult, rawStackConfigs, nil +} + +// ProcessYAMLConfigFile takes a path to a YAML stack manifest, +// recursively processes and deep-merges all imports, +// and returns the final stack config +func ProcessYAMLConfigFile( + cliConfig schema.CliConfiguration, + basePath string, + filePath string, + importsConfig map[string]map[any]any, + context map[string]any, + ignoreMissingFiles bool, + skipTemplatesProcessingInImports bool, + ignoreMissingTemplateValues bool, + skipIfMissing bool, + parentTerraformOverrides map[any]any, + parentHelmfileOverrides map[any]any, + atmosManifestJsonSchemaFilePath string, +) ( + map[any]any, + map[string]map[any]any, + map[any]any, + error, +) { + + var stackConfigs []map[any]any + relativeFilePath := u.TrimBasePathFromPath(basePath+"/", filePath) + + globalTerraformSection := map[any]any{} + globalHelmfileSection := map[any]any{} + globalOverrides := map[any]any{} + terraformOverrides := map[any]any{} + helmfileOverrides := map[any]any{} + finalTerraformOverrides := map[any]any{} + finalHelmfileOverrides := map[any]any{} + + stackYamlConfig, err := GetFileContent(filePath) + + // If the file does not exist (`err != nil`), and `ignoreMissingFiles = true`, don't return the error. + // + // `ignoreMissingFiles = true` is used when executing `atmos describe affected` command. + // If we add a new stack manifest with some component configurations to the current branch, then the new file will not be present in + // the remote branch (with which the current branch is compared), and Atmos would throw an error. + // + // `skipIfMissing` is used in Atmos imports (https://atmos.tools/core-concepts/stacks/imports). + // Set it to `true` to ignore the imported manifest if it does not exist, and don't throw an error. + // This is useful when generating Atmos manifests using other tools, but the imported files are not present yet at the generation time. + if err != nil { + if ignoreMissingFiles || skipIfMissing { + return map[any]any{}, map[string]map[any]any{}, map[any]any{}, nil + } else { + return nil, nil, nil, err + } + } + + stackManifestTemplatesProcessed := stackYamlConfig + stackManifestTemplatesErrorMessage := "" + + // Process `Go` templates in the imported stack manifest using the provided `context` + // https://atmos.tools/core-concepts/stacks/imports#go-templates-in-imports + if !skipTemplatesProcessingInImports && len(context) > 0 { + stackManifestTemplatesProcessed, err = ProcessTmpl(relativeFilePath, stackYamlConfig, context, ignoreMissingTemplateValues) + if err != nil { + if cliConfig.Logs.Level == u.LogLevelTrace || cliConfig.Logs.Level == u.LogLevelDebug { + stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) + } + e := fmt.Errorf("invalid stack manifest '%s'\n%v%s", relativeFilePath, err, stackManifestTemplatesErrorMessage) + return nil, nil, nil, e + } + } + + stackConfigMap, err := c.YAMLToMapOfInterfaces(stackManifestTemplatesProcessed) + if err != nil { + if cliConfig.Logs.Level == u.LogLevelTrace || cliConfig.Logs.Level == u.LogLevelDebug { + stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) + } + e := fmt.Errorf("invalid stack manifest '%s'\n%v%s", relativeFilePath, err, stackManifestTemplatesErrorMessage) + return nil, nil, nil, e + } + + // If the path to the Atmos manifest JSON Schema is provided, validate the stack manifest against it + if atmosManifestJsonSchemaFilePath != "" { + // Convert the data to JSON and back to Go map to prevent the error: + // jsonschema: invalid jsonType: map[interface {}]interface {} + dataJson, err := u.ConvertToJSONFast(stackConfigMap) + if err != nil { + return nil, nil, nil, err + } + + dataFromJson, err := u.ConvertFromJSON(dataJson) + if err != nil { + return nil, nil, nil, err + } + + compiler := jsonschema.NewCompiler() + + atmosManifestJsonSchemaValidationErrorFormat := "Atmos manifest JSON Schema validation error in the file '%s':\n%v" + + atmosManifestJsonSchemaFileReader, err := os.Open(atmosManifestJsonSchemaFilePath) + if err != nil { + return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) + } + + if err := compiler.AddResource(atmosManifestJsonSchemaFilePath, atmosManifestJsonSchemaFileReader); err != nil { + return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) + } + + compiler.Draft = jsonschema.Draft2020 + + compiledSchema, err := compiler.Compile(atmosManifestJsonSchemaFilePath) + if err != nil { + return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) + } + + if err = compiledSchema.Validate(dataFromJson); err != nil { + switch e := err.(type) { + case *jsonschema.ValidationError: + b, err2 := json.MarshalIndent(e.BasicOutput(), "", " ") + if err2 != nil { + return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err2) + } + return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, string(b)) + default: + return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) + } + } + } + + // Check if the `overrides` sections exist and if we need to process overrides for the components in this stack manifest and its imports + + // Global overrides + if i, ok := stackConfigMap[cfg.OverridesSectionName]; ok { + if globalOverrides, ok = i.(map[any]any); !ok { + return nil, nil, nil, fmt.Errorf("invalid 'overrides' section in the stack manifest '%s'", relativeFilePath) + } + } + + // Terraform overrides + if o, ok := stackConfigMap["terraform"]; ok { + if globalTerraformSection, ok = o.(map[any]any); !ok { + return nil, nil, nil, fmt.Errorf("invalid 'terraform' section in the stack manifest '%s'", relativeFilePath) + } + + if i, ok := globalTerraformSection[cfg.OverridesSectionName]; ok { + if terraformOverrides, ok = i.(map[any]any); !ok { + return nil, nil, nil, fmt.Errorf("invalid 'terraform.overrides' section in the stack manifest '%s'", relativeFilePath) + } + } + } + + finalTerraformOverrides, err = m.Merge( + cliConfig, + []map[any]any{globalOverrides, terraformOverrides, parentTerraformOverrides}, + ) + if err != nil { + return nil, nil, nil, err + } + + // Helmfile overrides + if o, ok := stackConfigMap["helmfile"]; ok { + if globalHelmfileSection, ok = o.(map[any]any); !ok { + return nil, nil, nil, fmt.Errorf("invalid 'helmfile' section in the stack manifest '%s'", relativeFilePath) + } + + if i, ok := globalHelmfileSection[cfg.OverridesSectionName]; ok { + if helmfileOverrides, ok = i.(map[any]any); !ok { + return nil, nil, nil, fmt.Errorf("invalid 'terraform.overrides' section in the stack manifest '%s'", relativeFilePath) + } + } + } + + finalHelmfileOverrides, err = m.Merge( + cliConfig, + []map[any]any{globalOverrides, helmfileOverrides, parentHelmfileOverrides}, + ) + if err != nil { + return nil, nil, nil, err + } + + // Add the `overrides` section for all components in this manifest + if len(finalTerraformOverrides) > 0 || len(finalHelmfileOverrides) > 0 { + if componentsSection, ok := stackConfigMap["components"].(map[any]any); ok { + // Terraform + if len(finalTerraformOverrides) > 0 { + if terraformSection, ok := componentsSection["terraform"].(map[any]any); ok { + for _, compSection := range terraformSection { + if componentSection, ok := compSection.(map[any]any); ok { + componentSection["overrides"] = finalTerraformOverrides + } + } + } + } + + // Helmfile + if len(finalHelmfileOverrides) > 0 { + if helmfileSection, ok := componentsSection["helmfile"].(map[any]any); ok { + for _, compSection := range helmfileSection { + if componentSection, ok := compSection.(map[any]any); ok { + componentSection["overrides"] = finalHelmfileOverrides + } + } + } + } + } + } + + // Find and process all imports + importStructs, err := ProcessImportSection(stackConfigMap, relativeFilePath) + if err != nil { + return nil, nil, nil, err + } + + for _, importStruct := range importStructs { + imp := importStruct.Path + + if imp == "" { + return nil, nil, nil, fmt.Errorf("invalid empty import in the manifest '%s'", relativeFilePath) + } + + // If the import file is specified without extension, use `.yaml` as default + impWithExt := imp + ext := filepath.Ext(imp) + if ext == "" { + ext = cfg.DefaultStackConfigFileExtension + impWithExt = imp + ext + } + + impWithExtPath := path.Join(basePath, impWithExt) + + if impWithExtPath == filePath { + errorMessage := fmt.Sprintf("invalid import in the manifest '%s'\nThe file imports itself in '%s'", + relativeFilePath, + imp) + return nil, nil, nil, errors.New(errorMessage) + } + + // Find all import matches in the glob + importMatches, err := u.GetGlobMatches(impWithExtPath) + if err != nil || len(importMatches) == 0 { + // Retry (b/c we are using `doublestar` library and it sometimes has issues reading many files in a Docker container) + // TODO: review `doublestar` library + + importMatches, err = u.GetGlobMatches(impWithExtPath) + if err != nil || len(importMatches) == 0 { + // The import was not found -> check if the import is a Go template; if not, return the error + isGolangTemplate, err2 := IsGolangTemplate(imp) + if err2 != nil { + return nil, nil, nil, err2 + } + + // If the import is not a Go template and SkipIfMissing is false, return the error + if !isGolangTemplate && !importStruct.SkipIfMissing { + if err != nil { + errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'\nError: %s", + imp, + relativeFilePath, + err, + ) + return nil, nil, nil, errors.New(errorMessage) + } else if importMatches == nil { + errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'", + imp, + relativeFilePath, + ) + return nil, nil, nil, errors.New(errorMessage) + } + } + } + } + + // Support `context` in hierarchical imports. + // Deep-merge the parent `context` with the current `context` and propagate the result to the entire chain of imports. + // The parent `context` takes precedence over the current (imported) `context` and will override items with the same keys. + // TODO: instead of calling the conversion functions, we need to switch to generics and update everything to support it + listOfMaps := []map[any]any{c.MapsOfStringsToMapsOfInterfaces(importStruct.Context), c.MapsOfStringsToMapsOfInterfaces(context)} + mergedContext, err := m.Merge(cliConfig, listOfMaps) + if err != nil { + return nil, nil, nil, err + } + + // Process the imports in the current manifest + for _, importFile := range importMatches { + yamlConfig, _, yamlConfigRaw, err := ProcessYAMLConfigFile( + cliConfig, + basePath, + importFile, + importsConfig, + c.MapsOfInterfacesToMapsOfStrings(mergedContext), + ignoreMissingFiles, + importStruct.SkipTemplatesProcessing, + true, // importStruct.IgnoreMissingTemplateValues, + importStruct.SkipIfMissing, + finalTerraformOverrides, + finalHelmfileOverrides, + "", + ) + if err != nil { + return nil, nil, nil, err + } + + stackConfigs = append(stackConfigs, yamlConfig) + importRelativePathWithExt := strings.Replace(importFile, basePath+"/", "", 1) + ext2 := filepath.Ext(importRelativePathWithExt) + if ext2 == "" { + ext2 = cfg.DefaultStackConfigFileExtension + } + importRelativePathWithoutExt := strings.TrimSuffix(importRelativePathWithExt, ext2) + importsConfig[importRelativePathWithoutExt] = yamlConfigRaw + } + } + + if len(stackConfigMap) > 0 { + stackConfigs = append(stackConfigs, stackConfigMap) + } + + // Deep-merge the stack manifest and all the imports + stackConfigsDeepMerged, err := m.Merge(cliConfig, stackConfigs) + if err != nil { + return nil, nil, nil, err + } + + return stackConfigsDeepMerged, importsConfig, stackConfigMap, nil +} + +// ProcessStackConfig takes a stack manifest, deep-merges all variables, settings, environments and backends, +// and returns the final stack configuration for all Terraform and helmfile components +func ProcessStackConfig( + cliConfig schema.CliConfiguration, + stacksBasePath string, + terraformComponentsBasePath string, + helmfileComponentsBasePath string, + stack string, + config map[any]any, + processStackDeps bool, + processComponentDeps bool, + componentTypeFilter string, + componentStackMap map[string]map[string][]string, + importsConfig map[string]map[any]any, + checkBaseComponentExists bool, +) (map[any]any, error) { + + stackName := strings.TrimSuffix( + strings.TrimSuffix( + u.TrimBasePathFromPath(stacksBasePath+"/", stack), + cfg.DefaultStackConfigFileExtension), + ".yml", + ) + + globalVarsSection := map[any]any{} + globalSettingsSection := map[any]any{} + globalEnvSection := map[any]any{} + globalTerraformSection := map[any]any{} + globalHelmfileSection := map[any]any{} + globalComponentsSection := map[any]any{} + + terraformVars := map[any]any{} + terraformSettings := map[any]any{} + terraformEnv := map[any]any{} + terraformCommand := "" + terraformProviders := map[any]any{} + + helmfileVars := map[any]any{} + helmfileSettings := map[any]any{} + helmfileEnv := map[any]any{} + helmfileCommand := "" + + terraformComponents := map[string]any{} + helmfileComponents := map[string]any{} + allComponents := map[string]any{} + + // Global sections + if i, ok := config["vars"]; ok { + globalVarsSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'vars' section in the file '%s'", stackName) + } + } + + if i, ok := config["settings"]; ok { + globalSettingsSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'settings' section in the file '%s'", stackName) + } + } + + if i, ok := config["env"]; ok { + globalEnvSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'env' section in the file '%s'", stackName) + } + } + + if i, ok := config["terraform"]; ok { + globalTerraformSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform' section in the file '%s'", stackName) + } + } + + if i, ok := config["helmfile"]; ok { + globalHelmfileSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'helmfile' section in the file '%s'", stackName) + } + } + + if i, ok := config["components"]; ok { + globalComponentsSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components' section in the file '%s'", stackName) + } + } + + // Terraform section + if i, ok := globalTerraformSection[cfg.CommandSectionName]; ok { + terraformCommand, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.command' section in the file '%s'", stackName) + } + } + + if i, ok := globalTerraformSection["vars"]; ok { + terraformVars, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.vars' section in the file '%s'", stackName) + } + } + + globalAndTerraformVars, err := m.Merge(cliConfig, []map[any]any{globalVarsSection, terraformVars}) + if err != nil { + return nil, err + } + + if i, ok := globalTerraformSection["settings"]; ok { + terraformSettings, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.settings' section in the file '%s'", stackName) + } + } + + globalAndTerraformSettings, err := m.Merge(cliConfig, []map[any]any{globalSettingsSection, terraformSettings}) + if err != nil { + return nil, err + } + + if i, ok := globalTerraformSection["env"]; ok { + terraformEnv, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.env' section in the file '%s'", stackName) + } + } + + globalAndTerraformEnv, err := m.Merge(cliConfig, []map[any]any{globalEnvSection, terraformEnv}) + if err != nil { + return nil, err + } + + if i, ok := globalTerraformSection[cfg.ProvidersSectionName]; ok { + terraformProviders, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.providers' section in the file '%s'", stackName) + } + } + + // Global backend + globalBackendType := "" + globalBackendSection := map[any]any{} + + if i, ok := globalTerraformSection["backend_type"]; ok { + globalBackendType, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.backend_type' section in the file '%s'", stackName) + } + } + + if i, ok := globalTerraformSection["backend"]; ok { + globalBackendSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.backend' section in the file '%s'", stackName) + } + } + + // Global remote state backend + globalRemoteStateBackendType := "" + globalRemoteStateBackendSection := map[any]any{} + + if i, ok := globalTerraformSection["remote_state_backend_type"]; ok { + globalRemoteStateBackendType, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.remote_state_backend_type' section in the file '%s'", stackName) + } + } + + if i, ok := globalTerraformSection["remote_state_backend"]; ok { + globalRemoteStateBackendSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.remote_state_backend' section in the file '%s'", stackName) + } + } + + // Helmfile section + if i, ok := globalHelmfileSection[cfg.CommandSectionName]; ok { + helmfileCommand, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'helmfile.command' section in the file '%s'", stackName) + } + } + + if i, ok := globalHelmfileSection["vars"]; ok { + helmfileVars, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'helmfile.vars' section in the file '%s'", stackName) + } + } + + globalAndHelmfileVars, err := m.Merge(cliConfig, []map[any]any{globalVarsSection, helmfileVars}) + if err != nil { + return nil, err + } + + if i, ok := globalHelmfileSection["settings"]; ok { + helmfileSettings, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'helmfile.settings' section in the file '%s'", stackName) + } + } + + globalAndHelmfileSettings, err := m.Merge(cliConfig, []map[any]any{globalSettingsSection, helmfileSettings}) + if err != nil { + return nil, err + } + + if i, ok := globalHelmfileSection["env"]; ok { + helmfileEnv, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'helmfile.env' section in the file '%s'", stackName) + } + } + + globalAndHelmfileEnv, err := m.Merge(cliConfig, []map[any]any{globalEnvSection, helmfileEnv}) + if err != nil { + return nil, err + } + + // Process all Terraform components + if componentTypeFilter == "" || componentTypeFilter == "terraform" { + if allTerraformComponents, ok := globalComponentsSection["terraform"]; ok { + + allTerraformComponentsMap, ok := allTerraformComponents.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform' section in the file '%s'", stackName) + } + + for cmp, v := range allTerraformComponentsMap { + component := cmp.(string) + + componentMap, ok := v.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s' section in the file '%s'", component, stackName) + } + + componentVars := map[any]any{} + if i, ok := componentMap[cfg.VarsSectionName]; ok { + componentVars, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.vars' section in the file '%s'", component, stackName) + } + } + + componentSettings := map[any]any{} + if i, ok := componentMap[cfg.SettingsSectionName]; ok { + componentSettings, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.settings' section in the file '%s'", component, stackName) + } + + if i, ok := componentSettings["spacelift"]; ok { + _, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.settings.spacelift' section in the file '%s'", component, stackName) + } + } + } + + componentEnv := map[any]any{} + if i, ok := componentMap[cfg.EnvSectionName]; ok { + componentEnv, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.env' section in the file '%s'", component, stackName) + } + } + + componentProviders := map[any]any{} + if i, ok := componentMap[cfg.ProvidersSectionName]; ok { + componentProviders, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.providers' section in the file '%s'", component, stackName) + } + } + + // Component metadata. + // This is per component, not deep-merged and not inherited from base components and globals. + componentMetadata := map[any]any{} + if i, ok := componentMap[cfg.MetadataSectionName]; ok { + componentMetadata, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.metadata' section in the file '%s'", component, stackName) + } + } + + // Component backend + componentBackendType := "" + componentBackendSection := map[any]any{} + + if i, ok := componentMap[cfg.BackendTypeSectionName]; ok { + componentBackendType, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.backend_type' attribute in the file '%s'", component, stackName) + } + } + + if i, ok := componentMap[cfg.BackendSectionName]; ok { + componentBackendSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.backend' section in the file '%s'", component, stackName) + } + } + + // Component remote state backend + componentRemoteStateBackendType := "" + componentRemoteStateBackendSection := map[any]any{} + + if i, ok := componentMap["remote_state_backend_type"]; ok { + componentRemoteStateBackendType, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.remote_state_backend_type' attribute in the file '%s'", component, stackName) + } + } + + if i, ok := componentMap["remote_state_backend"]; ok { + componentRemoteStateBackendSection, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.remote_state_backend' section in the file '%s'", component, stackName) + } + } + + componentTerraformCommand := "" + if i, ok := componentMap[cfg.CommandSectionName]; ok { + componentTerraformCommand, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.command' attribute in the file '%s'", component, stackName) + } + } + + // Process overrides + componentOverrides := map[any]any{} + componentOverridesVars := map[any]any{} + componentOverridesSettings := map[any]any{} + componentOverridesEnv := map[any]any{} + componentOverridesProviders := map[any]any{} + componentOverridesTerraformCommand := "" + + if i, ok := componentMap[cfg.OverridesSectionName]; ok { + if componentOverrides, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides' in the manifest '%s'", component, stackName) + } + + if i, ok = componentOverrides[cfg.VarsSectionName]; ok { + if componentOverridesVars, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.vars' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.SettingsSectionName]; ok { + if componentOverridesSettings, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.settings' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.EnvSectionName]; ok { + if componentOverridesEnv, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.env' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.CommandSectionName]; ok { + if componentOverridesTerraformCommand, ok = i.(string); !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.command' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.ProvidersSectionName]; ok { + if componentOverridesProviders, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.providers' in the manifest '%s'", component, stackName) + } + } + } + + // Process base component(s) + baseComponentName := "" + baseComponentVars := map[any]any{} + baseComponentSettings := map[any]any{} + baseComponentEnv := map[any]any{} + baseComponentProviders := map[any]any{} + baseComponentTerraformCommand := "" + baseComponentBackendType := "" + baseComponentBackendSection := map[any]any{} + baseComponentRemoteStateBackendType := "" + baseComponentRemoteStateBackendSection := map[any]any{} + var baseComponentConfig schema.BaseComponentConfig + var componentInheritanceChain []string + var baseComponents []string + + // Inheritance using the top-level `component` attribute + if baseComponent, baseComponentExist := componentMap[cfg.ComponentSectionName]; baseComponentExist { + baseComponentName, ok = baseComponent.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.component' attribute in the file '%s'", component, stackName) + } + + // Process the base components recursively to find `componentInheritanceChain` + err = ProcessBaseComponentConfig( + cliConfig, + &baseComponentConfig, + allTerraformComponentsMap, + component, + stack, + baseComponentName, + terraformComponentsBasePath, + checkBaseComponentExists, + &baseComponents, + ) + if err != nil { + return nil, err + } + + baseComponentVars = baseComponentConfig.BaseComponentVars + baseComponentSettings = baseComponentConfig.BaseComponentSettings + baseComponentEnv = baseComponentConfig.BaseComponentEnv + baseComponentProviders = baseComponentConfig.BaseComponentProviders + baseComponentName = baseComponentConfig.FinalBaseComponentName + baseComponentTerraformCommand = baseComponentConfig.BaseComponentCommand + baseComponentBackendType = baseComponentConfig.BaseComponentBackendType + baseComponentBackendSection = baseComponentConfig.BaseComponentBackendSection + baseComponentRemoteStateBackendType = baseComponentConfig.BaseComponentRemoteStateBackendType + baseComponentRemoteStateBackendSection = baseComponentConfig.BaseComponentRemoteStateBackendSection + componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain + } + + // Multiple inheritance (and multiple-inheritance chain) using `metadata.component` and `metadata.inherit`. + // `metadata.component` points to the component implementation (e.g. in `components/terraform` folder), + // it does not specify inheritance (it overrides the deprecated top-level `component` attribute). + // `metadata.inherit` is a list of component names from which the current component inherits. + // It uses a method similar to Method Resolution Order (MRO), which is how Python supports multiple inheritance. + // + // In the case of multiple base components, it is processed left to right, in the order by which it was declared. + // For example: `metadata.inherits: [componentA, componentB]` + // will deep-merge all the base components of `componentA` (each component overriding its base), + // then all the base components of `componentB` (each component overriding its base), + // then the two results are deep-merged together (`componentB` inheritance chain will override values from 'componentA' inheritance chain). + if baseComponentFromMetadata, baseComponentFromMetadataExist := componentMetadata[cfg.ComponentSectionName]; baseComponentFromMetadataExist { + baseComponentName, ok = baseComponentFromMetadata.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.metadata.component' attribute in the file '%s'", component, stackName) + } + } + + baseComponents = append(baseComponents, baseComponentName) + + if inheritList, inheritListExist := componentMetadata["inherits"].([]any); inheritListExist { + for _, v := range inheritList { + baseComponentFromInheritList, ok := v.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.metadata.inherits' section in the file '%s'", component, stackName) + } + + if _, ok := allTerraformComponentsMap[baseComponentFromInheritList]; !ok { + if checkBaseComponentExists { + errorMessage := fmt.Sprintf("The component '%[1]s' in the stack manifest '%[2]s' inherits from '%[3]s' "+ + "(using 'metadata.inherits'), but '%[3]s' is not defined in any of the config files for the stack '%[2]s'", + component, + stackName, + baseComponentFromInheritList, + ) + return nil, errors.New(errorMessage) + } + } + + // Process the baseComponentFromInheritList components recursively to find `componentInheritanceChain` + err = ProcessBaseComponentConfig( + cliConfig, + &baseComponentConfig, + allTerraformComponentsMap, + component, + stack, + baseComponentFromInheritList, + terraformComponentsBasePath, + checkBaseComponentExists, + &baseComponents, + ) + if err != nil { + return nil, err + } + + baseComponentVars = baseComponentConfig.BaseComponentVars + baseComponentSettings = baseComponentConfig.BaseComponentSettings + baseComponentEnv = baseComponentConfig.BaseComponentEnv + baseComponentTerraformCommand = baseComponentConfig.BaseComponentCommand + baseComponentBackendType = baseComponentConfig.BaseComponentBackendType + baseComponentBackendSection = baseComponentConfig.BaseComponentBackendSection + baseComponentRemoteStateBackendType = baseComponentConfig.BaseComponentRemoteStateBackendType + baseComponentRemoteStateBackendSection = baseComponentConfig.BaseComponentRemoteStateBackendSection + componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain + } + } + + baseComponents = u.UniqueStrings(baseComponents) + sort.Strings(baseComponents) + + // Final configs + finalComponentVars, err := m.Merge( + cliConfig, + []map[any]any{ + globalAndTerraformVars, + baseComponentVars, + componentVars, + componentOverridesVars, + }) + if err != nil { + return nil, err + } + + finalComponentSettings, err := m.Merge( + cliConfig, + []map[any]any{ + globalAndTerraformSettings, + baseComponentSettings, + componentSettings, + componentOverridesSettings, + }) + if err != nil { + return nil, err + } + + finalComponentEnv, err := m.Merge( + cliConfig, + []map[any]any{ + globalAndTerraformEnv, + baseComponentEnv, + componentEnv, + componentOverridesEnv, + }) + if err != nil { + return nil, err + } + + finalComponentProviders, err := m.Merge( + cliConfig, + []map[any]any{ + terraformProviders, + baseComponentProviders, + componentProviders, + componentOverridesProviders, + }) + if err != nil { + return nil, err + } + + // Final backend + finalComponentBackendType := globalBackendType + if len(baseComponentBackendType) > 0 { + finalComponentBackendType = baseComponentBackendType + } + if len(componentBackendType) > 0 { + finalComponentBackendType = componentBackendType + } + + finalComponentBackendSection, err := m.Merge( + cliConfig, + []map[any]any{ + globalBackendSection, + baseComponentBackendSection, + componentBackendSection, + }) + if err != nil { + return nil, err + } + + finalComponentBackend := map[any]any{} + if i, ok := finalComponentBackendSection[finalComponentBackendType]; ok { + finalComponentBackend, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.backend' section for the component '%s'", component) + } + } + + // AWS S3 backend + // Check if `backend` section has `workspace_key_prefix` for `s3` backend type + // If it does not, use the component name instead + // It will also be propagated to `remote_state_backend` section of `s3` type + if finalComponentBackendType == "s3" { + if p, ok := finalComponentBackend["workspace_key_prefix"].(string); !ok || p == "" { + workspaceKeyPrefix := component + if baseComponentName != "" { + workspaceKeyPrefix = baseComponentName + } + finalComponentBackend["workspace_key_prefix"] = strings.Replace(workspaceKeyPrefix, "/", "-", -1) + } + } + + // Google GSC backend + // Check if `backend` section has `prefix` for `gcs` backend type + // If it does not, use the component name instead + // https://developer.hashicorp.com/terraform/language/settings/backends/gcs + // https://developer.hashicorp.com/terraform/language/settings/backends/gcs#prefix + if finalComponentBackendType == "gcs" { + if p, ok := finalComponentBackend["prefix"].(string); !ok || p == "" { + prefix := component + if baseComponentName != "" { + prefix = baseComponentName + } + finalComponentBackend["prefix"] = strings.Replace(prefix, "/", "-", -1) + } + } + + // Azure backend + // Check if component `backend` section has `key` for `azurerm` backend type + // If it does not, use the component name instead and format it with the global backend key name to auto generate a unique Terraform state key + // The backend state file will be formatted like so: {global key name}/{component name}.terraform.tfstate + if finalComponentBackendType == "azurerm" { + if componentAzurerm, componentAzurermExists := componentBackendSection["azurerm"].(map[any]any); !componentAzurermExists { + if _, componentAzurermKeyExists := componentAzurerm["key"].(string); !componentAzurermKeyExists { + azureKeyPrefixComponent := component + var keyName []string + if baseComponentName != "" { + azureKeyPrefixComponent = baseComponentName + } + if globalAzurerm, globalAzurermExists := globalBackendSection["azurerm"].(map[any]any); globalAzurermExists { + if _, globalAzurermKeyExists := globalAzurerm["key"].(string); globalAzurermKeyExists { + keyName = append(keyName, globalAzurerm["key"].(string)) + } + } + componentKeyName := strings.Replace(azureKeyPrefixComponent, "/", "-", -1) + keyName = append(keyName, fmt.Sprintf("%s.terraform.tfstate", componentKeyName)) + finalComponentBackend["key"] = strings.Join(keyName, "/") + } + } + } + + // Final remote state backend + finalComponentRemoteStateBackendType := finalComponentBackendType + if len(globalRemoteStateBackendType) > 0 { + finalComponentRemoteStateBackendType = globalRemoteStateBackendType + } + if len(baseComponentRemoteStateBackendType) > 0 { + finalComponentRemoteStateBackendType = baseComponentRemoteStateBackendType + } + if len(componentRemoteStateBackendType) > 0 { + finalComponentRemoteStateBackendType = componentRemoteStateBackendType + } + + finalComponentRemoteStateBackendSection, err := m.Merge( + cliConfig, + []map[any]any{ + globalRemoteStateBackendSection, + baseComponentRemoteStateBackendSection, + componentRemoteStateBackendSection, + }) + if err != nil { + return nil, err + } + + // Merge `backend` and `remote_state_backend` sections + // This will allow keeping `remote_state_backend` section DRY + finalComponentRemoteStateBackendSectionMerged, err := m.Merge( + cliConfig, + []map[any]any{ + finalComponentBackendSection, + finalComponentRemoteStateBackendSection, + }) + if err != nil { + return nil, err + } + + finalComponentRemoteStateBackend := map[any]any{} + if i, ok := finalComponentRemoteStateBackendSectionMerged[finalComponentRemoteStateBackendType]; ok { + finalComponentRemoteStateBackend, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'terraform.remote_state_backend' section for the component '%s'", component) + } + } + + // Final binary to execute + // Check for the binary in the following order: + // - `components.terraform.command` section in `atmos.yaml` CLI config file + // - global `terraform.command` section + // - base component(s) `command` section + // - component `command` section + // - `overrides.command` section + finalComponentTerraformCommand := "terraform" + if cliConfig.Components.Terraform.Command != "" { + finalComponentTerraformCommand = cliConfig.Components.Terraform.Command + } + if terraformCommand != "" { + finalComponentTerraformCommand = terraformCommand + } + if baseComponentTerraformCommand != "" { + finalComponentTerraformCommand = baseComponentTerraformCommand + } + if componentTerraformCommand != "" { + finalComponentTerraformCommand = componentTerraformCommand + } + if componentOverridesTerraformCommand != "" { + finalComponentTerraformCommand = componentOverridesTerraformCommand + } + + // If the component is not deployable (`metadata.type: abstract`), remove `settings.spacelift.workspace_enabled` from the map). + // This will prevent the derived components from inheriting `settings.spacelift.workspace_enabled=false` of not-deployable components. + // Also, removing `settings.spacelift.workspace_enabled` will effectively make it `false` + // and `spacelift_stack_processor` will not create a Spacelift stack for the abstract component + // even if `settings.spacelift.workspace_enabled` was set to `true`. + // This is per component, not deep-merged and not inherited from base components and globals. + componentIsAbstract := false + if componentType, componentTypeAttributeExists := componentMetadata["type"].(string); componentTypeAttributeExists { + if componentType == "abstract" { + componentIsAbstract = true + } + } + if componentIsAbstract { + if i, ok := finalComponentSettings["spacelift"]; ok { + spaceliftSettings, ok := i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.terraform.%s.settings.spacelift' section in the file '%s'", component, stackName) + } + delete(spaceliftSettings, "workspace_enabled") + } + } + + comp := map[string]any{} + comp[cfg.VarsSectionName] = finalComponentVars + comp[cfg.SettingsSectionName] = finalComponentSettings + comp[cfg.EnvSectionName] = finalComponentEnv + comp[cfg.BackendTypeSectionName] = finalComponentBackendType + comp[cfg.BackendSectionName] = finalComponentBackend + comp["remote_state_backend_type"] = finalComponentRemoteStateBackendType + comp["remote_state_backend"] = finalComponentRemoteStateBackend + comp[cfg.CommandSectionName] = finalComponentTerraformCommand + comp["inheritance"] = componentInheritanceChain + comp[cfg.MetadataSectionName] = componentMetadata + comp[cfg.OverridesSectionName] = componentOverrides + comp[cfg.ProvidersSectionName] = finalComponentProviders + + if baseComponentName != "" { + comp[cfg.ComponentSectionName] = baseComponentName + } + + terraformComponents[component] = comp + } + } + } + + // Process all helmfile components + if componentTypeFilter == "" || componentTypeFilter == "helmfile" { + if allHelmfileComponents, ok := globalComponentsSection["helmfile"]; ok { + + allHelmfileComponentsMap, ok := allHelmfileComponents.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile' section in the file '%s'", stackName) + } + + for cmp, v := range allHelmfileComponentsMap { + component := cmp.(string) + + componentMap, ok := v.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s' section in the file '%s'", component, stackName) + } + + componentVars := map[any]any{} + if i2, ok := componentMap[cfg.VarsSectionName]; ok { + componentVars, ok = i2.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.vars' section in the file '%s'", component, stackName) + } + } + + componentSettings := map[any]any{} + if i, ok := componentMap[cfg.SettingsSectionName]; ok { + componentSettings, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.settings' section in the file '%s'", component, stackName) + } + } + + componentEnv := map[any]any{} + if i, ok := componentMap[cfg.EnvSectionName]; ok { + componentEnv, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.env' section in the file '%s'", component, stackName) + } + } + + // Component metadata. + // This is per component, not deep-merged and not inherited from base components and globals. + componentMetadata := map[any]any{} + if i, ok := componentMap[cfg.MetadataSectionName]; ok { + componentMetadata, ok = i.(map[any]any) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.metadata' section in the file '%s'", component, stackName) + } + } + + componentHelmfileCommand := "" + if i, ok := componentMap[cfg.CommandSectionName]; ok { + componentHelmfileCommand, ok = i.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.command' attribute in the file '%s'", component, stackName) + } + } + + // Process overrides + componentOverrides := map[any]any{} + componentOverridesVars := map[any]any{} + componentOverridesSettings := map[any]any{} + componentOverridesEnv := map[any]any{} + componentOverridesHelmfileCommand := "" + + if i, ok := componentMap[cfg.OverridesSectionName]; ok { + if componentOverrides, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides' in the manifest '%s'", component, stackName) + } + + if i, ok = componentOverrides[cfg.VarsSectionName]; ok { + if componentOverridesVars, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.vars' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.SettingsSectionName]; ok { + if componentOverridesSettings, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.settings' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.EnvSectionName]; ok { + if componentOverridesEnv, ok = i.(map[any]any); !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.env' in the manifest '%s'", component, stackName) + } + } + + if i, ok = componentOverrides[cfg.CommandSectionName]; ok { + if componentOverridesHelmfileCommand, ok = i.(string); !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.command' in the manifest '%s'", component, stackName) + } + } + } + + // Process base component(s) + baseComponentVars := map[any]any{} + baseComponentSettings := map[any]any{} + baseComponentEnv := map[any]any{} + baseComponentName := "" + baseComponentHelmfileCommand := "" + var baseComponentConfig schema.BaseComponentConfig + var componentInheritanceChain []string + var baseComponents []string + + // Inheritance using the top-level `component` attribute + if baseComponent, baseComponentExist := componentMap[cfg.ComponentSectionName]; baseComponentExist { + baseComponentName, ok = baseComponent.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.component' attribute in the file '%s'", component, stackName) + } + + // Process the base components recursively to find `componentInheritanceChain` + err = ProcessBaseComponentConfig( + cliConfig, + &baseComponentConfig, + allHelmfileComponentsMap, + component, + stack, + baseComponentName, + helmfileComponentsBasePath, + checkBaseComponentExists, + &baseComponents, + ) + if err != nil { + return nil, err + } + + baseComponentVars = baseComponentConfig.BaseComponentVars + baseComponentSettings = baseComponentConfig.BaseComponentSettings + baseComponentEnv = baseComponentConfig.BaseComponentEnv + baseComponentName = baseComponentConfig.FinalBaseComponentName + baseComponentHelmfileCommand = baseComponentConfig.BaseComponentCommand + componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain + } + + // Multiple inheritance (and multiple-inheritance chain) using `metadata.component` and `metadata.inherit`. + // `metadata.component` points to the component implementation (e.g. in `components/terraform` folder), + // it does not specify inheritance (it overrides the deprecated top-level `component` attribute). + // `metadata.inherit` is a list of component names from which the current component inherits. + // It uses a method similar to Method Resolution Order (MRO), which is how Python supports multiple inheritance. + // + // In the case of multiple base components, it is processed left to right, in the order by which it was declared. + // For example: `metadata.inherits: [componentA, componentB]` + // will deep-merge all the base components of `componentA` (each component overriding its base), + // then all the base components of `componentB` (each component overriding its base), + // then the two results are deep-merged together (`componentB` inheritance chain will override values from 'componentA' inheritance chain). + if baseComponentFromMetadata, baseComponentFromMetadataExist := componentMetadata[cfg.ComponentSectionName]; baseComponentFromMetadataExist { + baseComponentName, ok = baseComponentFromMetadata.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.metadata.component' attribute in the file '%s'", component, stackName) + } + } + + baseComponents = append(baseComponents, baseComponentName) + + if inheritList, inheritListExist := componentMetadata["inherits"].([]any); inheritListExist { + for _, v := range inheritList { + baseComponentFromInheritList, ok := v.(string) + if !ok { + return nil, fmt.Errorf("invalid 'components.helmfile.%s.metadata.inherits' section in the file '%s'", component, stackName) + } + + if _, ok := allHelmfileComponentsMap[baseComponentFromInheritList]; !ok { + if checkBaseComponentExists { + errorMessage := fmt.Sprintf("The component '%[1]s' in the stack manifest '%[2]s' inherits from '%[3]s' "+ + "(using 'metadata.inherits'), but '%[3]s' is not defined in any of the config files for the stack '%[2]s'", + component, + stackName, + baseComponentFromInheritList, + ) + return nil, errors.New(errorMessage) + } + } + + // Process the baseComponentFromInheritList components recursively to find `componentInheritanceChain` + err = ProcessBaseComponentConfig( + cliConfig, + &baseComponentConfig, + allHelmfileComponentsMap, + component, + stack, + baseComponentFromInheritList, + helmfileComponentsBasePath, + checkBaseComponentExists, + &baseComponents, + ) + if err != nil { + return nil, err + } + + baseComponentVars = baseComponentConfig.BaseComponentVars + baseComponentSettings = baseComponentConfig.BaseComponentSettings + baseComponentEnv = baseComponentConfig.BaseComponentEnv + baseComponentName = baseComponentConfig.FinalBaseComponentName + baseComponentHelmfileCommand = baseComponentConfig.BaseComponentCommand + componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain + } + } + + baseComponents = u.UniqueStrings(baseComponents) + sort.Strings(baseComponents) + + // Final configs + finalComponentVars, err := m.Merge( + cliConfig, + []map[any]any{ + globalAndHelmfileVars, + baseComponentVars, + componentVars, + componentOverridesVars, + }) + if err != nil { + return nil, err + } + + finalComponentSettings, err := m.Merge( + cliConfig, + []map[any]any{ + globalAndHelmfileSettings, + baseComponentSettings, + componentSettings, + componentOverridesSettings, + }) + if err != nil { + return nil, err + } + + finalComponentEnv, err := m.Merge( + cliConfig, + []map[any]any{ + globalAndHelmfileEnv, + baseComponentEnv, + componentEnv, + componentOverridesEnv, + }) + if err != nil { + return nil, err + } + + // Final binary to execute + // Check for the binary in the following order: + // - `components.helmfile.command` section in `atmos.yaml` CLI config file + // - global `helmfile.command` section + // - base component(s) `command` section + // - component `command` section + // - `overrides.command` section + finalComponentHelmfileCommand := "helmfile" + if cliConfig.Components.Helmfile.Command != "" { + finalComponentHelmfileCommand = cliConfig.Components.Helmfile.Command + } + if helmfileCommand != "" { + finalComponentHelmfileCommand = helmfileCommand + } + if baseComponentHelmfileCommand != "" { + finalComponentHelmfileCommand = baseComponentHelmfileCommand + } + if componentHelmfileCommand != "" { + finalComponentHelmfileCommand = componentHelmfileCommand + } + if componentOverridesHelmfileCommand != "" { + finalComponentHelmfileCommand = componentOverridesHelmfileCommand + } + + comp := map[string]any{} + comp[cfg.VarsSectionName] = finalComponentVars + comp[cfg.SettingsSectionName] = finalComponentSettings + comp[cfg.EnvSectionName] = finalComponentEnv + comp[cfg.CommandSectionName] = finalComponentHelmfileCommand + comp["inheritance"] = componentInheritanceChain + comp[cfg.MetadataSectionName] = componentMetadata + comp[cfg.OverridesSectionName] = componentOverrides + + if baseComponentName != "" { + comp[cfg.ComponentSectionName] = baseComponentName + } + + helmfileComponents[component] = comp + } + } + } + + allComponents["terraform"] = terraformComponents + allComponents["helmfile"] = helmfileComponents + + result := map[any]any{ + "components": allComponents, + } + + return result, nil +} + +// FindComponentStacks finds all infrastructure stack manifests where the component or the base component is defined +func FindComponentStacks( + componentType string, + component string, + baseComponent string, + componentStackMap map[string]map[string][]string) ([]string, error) { + + var stacks []string + + if componentStackConfig, componentStackConfigExists := componentStackMap[componentType]; componentStackConfigExists { + if componentStacks, componentStacksExist := componentStackConfig[component]; componentStacksExist { + stacks = append(stacks, componentStacks...) + } + + if baseComponent != "" { + if baseComponentStacks, baseComponentStacksExist := componentStackConfig[baseComponent]; baseComponentStacksExist { + stacks = append(stacks, baseComponentStacks...) + } + } + } + + unique := u.UniqueStrings(stacks) + sort.Strings(unique) + return unique, nil +} + +// FindComponentDependenciesLegacy finds all imports where the component or the base component(s) are defined +// Component depends on the imported config file if any of the following conditions is true: +// 1. The imported config file has any of the global `backend`, `backend_type`, `env`, `remote_state_backend`, `remote_state_backend_type`, +// `settings` or `vars` sections which are not empty. +// 2. The imported config file has the component type section, which has any of the `backend`, `backend_type`, `env`, `remote_state_backend`, +// `remote_state_backend_type`, `settings` or `vars` sections which are not empty. +// 3. The imported config file has the "components" section, which has the component type section, which has the component section. +// 4. The imported config file has the "components" section, which has the component type section, which has the base component(s) section, +// and the base component section is defined inline (not imported). +func FindComponentDependenciesLegacy( + stack string, + componentType string, + component string, + baseComponents []string, + stackImports map[string]map[any]any) ([]string, error) { + + var deps []string + + sectionsToCheck := []string{ + "backend", + "backend_type", + "env", + "remote_state_backend", + "remote_state_backend_type", + "settings", + "vars", + } + + for stackImportName, stackImportMap := range stackImports { + + if sectionContainsAnyNotEmptySections(stackImportMap, sectionsToCheck) { + deps = append(deps, stackImportName) + continue + } + + if sectionContainsAnyNotEmptySections(stackImportMap, []string{componentType}) { + if sectionContainsAnyNotEmptySections(stackImportMap[componentType].(map[any]any), sectionsToCheck) { + deps = append(deps, stackImportName) + continue + } + } + + stackImportMapComponentsSection, ok := stackImportMap["components"].(map[any]any) + if !ok { + continue + } + + stackImportMapComponentTypeSection, ok := stackImportMapComponentsSection[componentType].(map[any]any) + if !ok { + continue + } + + if stackImportMapComponentSection, ok := stackImportMapComponentTypeSection[component].(map[any]any); ok { + if len(stackImportMapComponentSection) > 0 { + deps = append(deps, stackImportName) + continue + } + } + + // Process base component(s) + // Only include the imported config file into "deps" if all the following conditions are `true`: + // 1. The imported config file has the base component(s) section(s) + // 2. The imported config file does not import other config files (which means that instead it defined the base component sections inline) + // 3. If the imported config file does import other config files, check that the base component sections in them are different by using + // `reflect.DeepEqual`. If they are the same, don't include the imported config file since it does not specify anything for the base component + for _, baseComponent := range baseComponents { + baseComponentSection, ok := stackImportMapComponentTypeSection[baseComponent].(map[any]any) + + if !ok || len(baseComponentSection) == 0 { + continue + } + + importOfStackImportStructs, err := ProcessImportSection(stackImportMap, stack) + if err != nil { + return nil, err + } + + if len(importOfStackImportStructs) == 0 { + deps = append(deps, stackImportName) + continue + } + + for _, importOfStackImportStruct := range importOfStackImportStructs { + importOfStackImportMap, ok := stackImports[importOfStackImportStruct.Path] + if !ok { + continue + } + + importOfStackImportComponentsSection, ok := importOfStackImportMap["components"].(map[any]any) + if !ok { + continue + } + + importOfStackImportComponentTypeSection, ok := importOfStackImportComponentsSection[componentType].(map[any]any) + if !ok { + continue + } + + importOfStackImportBaseComponentSection, ok := importOfStackImportComponentTypeSection[baseComponent].(map[any]any) + if !ok { + continue + } + + if !reflect.DeepEqual(baseComponentSection, importOfStackImportBaseComponentSection) { + deps = append(deps, stackImportName) + break + } + } + } + } + + deps = append(deps, stack) + unique := u.UniqueStrings(deps) + sort.Strings(unique) + return unique, nil +} + +// ProcessImportSection processes the `import` section in stack manifests +// The `import` section` can be of the following types: +// 1. list of `StackImport` structs +// 2. list of strings +// 3. List of strings and `StackImport` structs in the same file +func ProcessImportSection(stackMap map[any]any, filePath string) ([]schema.StackImport, error) { + stackImports, ok := stackMap[cfg.ImportSectionName] + + // If the stack file does not have the `import` section, return + if !ok || stackImports == nil { + return nil, nil + } + + // Check if the `import` section is a list of objects + importsList, ok := stackImports.([]any) + if !ok || len(importsList) == 0 { + return nil, fmt.Errorf("invalid 'import' section in the file '%s'", filePath) + } + + var result []schema.StackImport + + for _, imp := range importsList { + if imp == nil { + return nil, fmt.Errorf("invalid import in the file '%s'", filePath) + } + + // 1. Try to decode the import as the `StackImport` struct + var importObj schema.StackImport + err := mapstructure.Decode(imp, &importObj) + if err == nil { + result = append(result, importObj) + continue + } + + // 2. Try to cast the import to a string + s, ok := imp.(string) + if !ok { + return nil, fmt.Errorf("invalid import '%v' in the file '%s'", imp, filePath) + } + if s == "" { + return nil, fmt.Errorf("invalid empty import in the file '%s'", filePath) + } + + result = append(result, schema.StackImport{Path: s}) + } + + return result, nil +} + +// sectionContainsAnyNotEmptySections checks if a section contains any of the provided low-level sections, and it's not empty +func sectionContainsAnyNotEmptySections(section map[any]any, sectionsToCheck []string) bool { + for _, s := range sectionsToCheck { + if len(s) > 0 { + if v, ok := section[s]; ok { + if v2, ok2 := v.(map[any]any); ok2 && len(v2) > 0 { + return true + } + if v2, ok2 := v.(string); ok2 && len(v2) > 0 { + return true + } + } + } + } + return false +} + +// CreateComponentStackMap accepts a config file and creates a map of component-stack dependencies +func CreateComponentStackMap( + cliConfig schema.CliConfiguration, + stacksBasePath string, + terraformComponentsBasePath string, + helmfileComponentsBasePath string, + filePath string, +) (map[string]map[string][]string, error) { + + stackComponentMap := map[string]map[string][]string{} + stackComponentMap["terraform"] = map[string][]string{} + stackComponentMap["helmfile"] = map[string][]string{} + + componentStackMap := map[string]map[string][]string{} + componentStackMap["terraform"] = map[string][]string{} + componentStackMap["helmfile"] = map[string][]string{} + + dir := path.Dir(filePath) + + err := filepath.Walk(dir, + func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + isDirectory, err := u.IsDirectory(p) + if err != nil { + return err + } + + isYaml := u.IsYaml(p) + + if !isDirectory && isYaml { + config, _, _, err := ProcessYAMLConfigFile( + cliConfig, + stacksBasePath, + p, + map[string]map[any]any{}, + nil, + false, + false, + false, + false, + map[any]any{}, + map[any]any{}, + "", + ) + if err != nil { + return err + } + + finalConfig, err := ProcessStackConfig( + cliConfig, + stacksBasePath, + terraformComponentsBasePath, + helmfileComponentsBasePath, + p, + config, + false, + false, + "", + nil, + nil, + true) + if err != nil { + return err + } + + if componentsConfig, componentsConfigExists := finalConfig["components"]; componentsConfigExists { + componentsSection := componentsConfig.(map[string]any) + stackName := strings.Replace(p, stacksBasePath+"/", "", 1) + + if terraformConfig, terraformConfigExists := componentsSection["terraform"]; terraformConfigExists { + terraformSection := terraformConfig.(map[string]any) + + for k := range terraformSection { + stackComponentMap["terraform"][stackName] = append(stackComponentMap["terraform"][stackName], k) + } + } + + if helmfileConfig, helmfileConfigExists := componentsSection["helmfile"]; helmfileConfigExists { + helmfileSection := helmfileConfig.(map[string]any) + + for k := range helmfileSection { + stackComponentMap["helmfile"][stackName] = append(stackComponentMap["helmfile"][stackName], k) + } + } + } + } + + return nil + }) + + if err != nil { + return nil, err + } + + for stack, components := range stackComponentMap["terraform"] { + for _, component := range components { + componentStackMap["terraform"][component] = append(componentStackMap["terraform"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1)) + } + } + + for stack, components := range stackComponentMap["helmfile"] { + for _, component := range components { + componentStackMap["helmfile"][component] = append(componentStackMap["helmfile"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1)) + } + } + + return componentStackMap, nil +} + +// GetFileContent tries to read and return the file content from the sync map if it exists in the map, +// otherwise it reads the file, stores its content in the map and returns the content +func GetFileContent(filePath string) (string, error) { + existingContent, found := getFileContentSyncMap.Load(filePath) + if found && existingContent != nil { + return fmt.Sprintf("%s", existingContent), nil + } + + content, err := os.ReadFile(filePath) + if err != nil { + return "", err + } + getFileContentSyncMap.Store(filePath, content) + + return string(content), nil +} + +// ProcessBaseComponentConfig processes base component(s) config +func ProcessBaseComponentConfig( + cliConfig schema.CliConfiguration, + baseComponentConfig *schema.BaseComponentConfig, + allComponentsMap map[any]any, + component string, + stack string, + baseComponent string, + componentBasePath string, + checkBaseComponentExists bool, + baseComponents *[]string, +) error { + + if component == baseComponent { + return nil + } + + var baseComponentVars map[any]any + var baseComponentSettings map[any]any + var baseComponentEnv map[any]any + var baseComponentProviders map[any]any + var baseComponentCommand string + var baseComponentBackendType string + var baseComponentBackendSection map[any]any + var baseComponentRemoteStateBackendType string + var baseComponentRemoteStateBackendSection map[any]any + var baseComponentMap map[any]any + var ok bool + + *baseComponents = append(*baseComponents, baseComponent) + + if baseComponentSection, baseComponentSectionExist := allComponentsMap[baseComponent]; baseComponentSectionExist { + baseComponentMap, ok = baseComponentSection.(map[any]any) + if !ok { + // Depending on the code and libraries, the section can have different map types: map[any]any or map[string]any + // We try to convert to both + baseComponentMapOfStrings, ok := baseComponentSection.(map[string]any) + if !ok { + return fmt.Errorf("invalid config for the base component '%s' of the component '%s' in the stack '%s'", + baseComponent, component, stack) + } + baseComponentMap = c.MapsOfStringsToMapsOfInterfaces(baseComponentMapOfStrings) + } + + // First, process the base component(s) of this base component + if baseComponentOfBaseComponent, baseComponentOfBaseComponentExist := baseComponentMap["component"]; baseComponentOfBaseComponentExist { + baseComponentOfBaseComponentString, ok := baseComponentOfBaseComponent.(string) + if !ok { + return fmt.Errorf("invalid 'component:' section of the component '%s' in the stack '%s'", + baseComponent, stack) + } + + err := ProcessBaseComponentConfig( + cliConfig, + baseComponentConfig, + allComponentsMap, + baseComponent, + stack, + baseComponentOfBaseComponentString, + componentBasePath, + checkBaseComponentExists, + baseComponents, + ) + + if err != nil { + return err + } + } + + // Base component metadata. + // This is per component, not deep-merged and not inherited from base components and globals. + componentMetadata := map[any]any{} + if i, ok := baseComponentMap["metadata"]; ok { + componentMetadata, ok = i.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.metadata' section in the stack '%s'", component, stack) + } + + if inheritList, inheritListExist := componentMetadata["inherits"].([]any); inheritListExist { + for _, v := range inheritList { + baseComponentFromInheritList, ok := v.(string) + if !ok { + return fmt.Errorf("invalid '%s.metadata.inherits' section in the stack '%s'", component, stack) + } + + if _, ok := allComponentsMap[baseComponentFromInheritList]; !ok { + if checkBaseComponentExists { + errorMessage := fmt.Sprintf("The component '%[1]s' in the stack manifest '%[2]s' inherits from '%[3]s' "+ + "(using 'metadata.inherits'), but '%[3]s' is not defined in any of the config files for the stack '%[2]s'", + component, + stack, + baseComponentFromInheritList, + ) + return errors.New(errorMessage) + } + } + + // Process the baseComponentFromInheritList components recursively to find `componentInheritanceChain` + err := ProcessBaseComponentConfig( + cliConfig, + baseComponentConfig, + allComponentsMap, + component, + stack, + baseComponentFromInheritList, + componentBasePath, + checkBaseComponentExists, + baseComponents, + ) + if err != nil { + return err + } + } + } + } + + if baseComponentVarsSection, baseComponentVarsSectionExist := baseComponentMap["vars"]; baseComponentVarsSectionExist { + baseComponentVars, ok = baseComponentVarsSection.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.vars' section in the stack '%s'", baseComponent, stack) + } + } + + if baseComponentSettingsSection, baseComponentSettingsSectionExist := baseComponentMap["settings"]; baseComponentSettingsSectionExist { + baseComponentSettings, ok = baseComponentSettingsSection.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.settings' section in the stack '%s'", baseComponent, stack) + } + } + + if baseComponentEnvSection, baseComponentEnvSectionExist := baseComponentMap["env"]; baseComponentEnvSectionExist { + baseComponentEnv, ok = baseComponentEnvSection.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.env' section in the stack '%s'", baseComponent, stack) + } + } + + if baseComponentProvidersSection, baseComponentProvidersSectionExist := baseComponentMap[cfg.ProvidersSectionName]; baseComponentProvidersSectionExist { + baseComponentProviders, ok = baseComponentProvidersSection.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.providers' section in the stack '%s'", baseComponent, stack) + } + } + + // Base component backend + if i, ok2 := baseComponentMap["backend_type"]; ok2 { + baseComponentBackendType, ok = i.(string) + if !ok { + return fmt.Errorf("invalid '%s.backend_type' section in the stack '%s'", baseComponent, stack) + } + } + + if i, ok2 := baseComponentMap["backend"]; ok2 { + baseComponentBackendSection, ok = i.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.backend' section in the stack '%s'", baseComponent, stack) + } + } + + // Base component remote state backend + if i, ok2 := baseComponentMap["remote_state_backend_type"]; ok2 { + baseComponentRemoteStateBackendType, ok = i.(string) + if !ok { + return fmt.Errorf("invalid '%s.remote_state_backend_type' section in the stack '%s'", baseComponent, stack) + } + } + + if i, ok2 := baseComponentMap["remote_state_backend"]; ok2 { + baseComponentRemoteStateBackendSection, ok = i.(map[any]any) + if !ok { + return fmt.Errorf("invalid '%s.remote_state_backend' section in the stack '%s'", baseComponent, stack) + } + } + + // Base component `command` + if baseComponentCommandSection, baseComponentCommandSectionExist := baseComponentMap[cfg.CommandSectionName]; baseComponentCommandSectionExist { + baseComponentCommand, ok = baseComponentCommandSection.(string) + if !ok { + return fmt.Errorf("invalid '%s.command' section in the stack '%s'", baseComponent, stack) + } + } + + if len(baseComponentConfig.FinalBaseComponentName) == 0 { + baseComponentConfig.FinalBaseComponentName = baseComponent + } + + // Base component `vars` + merged, err := m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentVars, baseComponentVars}) + if err != nil { + return err + } + baseComponentConfig.BaseComponentVars = merged + + // Base component `settings` + merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentSettings, baseComponentSettings}) + if err != nil { + return err + } + baseComponentConfig.BaseComponentSettings = merged + + // Base component `env` + merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentEnv, baseComponentEnv}) + if err != nil { + return err + } + baseComponentConfig.BaseComponentEnv = merged + + // Base component `providers` + merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentProviders, baseComponentProviders}) + if err != nil { + return err + } + baseComponentConfig.BaseComponentProviders = merged + + // Base component `command` + baseComponentConfig.BaseComponentCommand = baseComponentCommand + + // Base component `backend_type` + baseComponentConfig.BaseComponentBackendType = baseComponentBackendType + + // Base component `backend` + merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentBackendSection, baseComponentBackendSection}) + if err != nil { + return err + } + baseComponentConfig.BaseComponentBackendSection = merged + + // Base component `remote_state_backend_type` + baseComponentConfig.BaseComponentRemoteStateBackendType = baseComponentRemoteStateBackendType + + // Base component `remote_state_backend` + merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentRemoteStateBackendSection, baseComponentRemoteStateBackendSection}) + if err != nil { + return err + } + baseComponentConfig.BaseComponentRemoteStateBackendSection = merged + + baseComponentConfig.ComponentInheritanceChain = u.UniqueStrings(append([]string{baseComponent}, baseComponentConfig.ComponentInheritanceChain...)) + } else { + if checkBaseComponentExists { + // Check if the base component exists as Terraform/Helmfile component + // If it does exist, don't throw errors if it is not defined in YAML config + componentPath := path.Join(componentBasePath, baseComponent) + componentPathExists, err := u.IsDirectory(componentPath) + if err != nil || !componentPathExists { + return errors.New("The component '" + component + "' inherits from the base component '" + + baseComponent + "' (using 'component:' attribute), " + "but `" + baseComponent + "' is not defined in any of the YAML config files for the stack '" + stack + "'") + } + } + } + + return nil +} + +// FindComponentsDerivedFromBaseComponents finds all components that derive from the given base components +func FindComponentsDerivedFromBaseComponents( + stack string, + allComponents map[string]any, + baseComponents []string, +) ([]string, error) { + + res := []string{} + + for component, compSection := range allComponents { + componentSection, ok := compSection.(map[string]any) + if !ok { + return nil, fmt.Errorf("invalid '%s' component section in the file '%s'", component, stack) + } + + if base, baseComponentExist := componentSection[cfg.ComponentSectionName]; baseComponentExist { + baseComponent, ok := base.(string) + if !ok { + return nil, fmt.Errorf("invalid 'component' attribute in the component '%s' in the file '%s'", component, stack) + } + + if baseComponent != "" && u.SliceContainsString(baseComponents, baseComponent) { + res = append(res, component) + } + } + } + + return res, nil +} diff --git a/internal/exec/stack_utils.go b/internal/exec/stack_utils.go index 547e8031d..d2c621f94 100644 --- a/internal/exec/stack_utils.go +++ b/internal/exec/stack_utils.go @@ -17,7 +17,7 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI var tmpl string if cliConfig.Stacks.NameTemplate != "" { - tmpl, err = u.ProcessTmpl("terraform-workspace-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + tmpl, err = ProcessTmpl("terraform-workspace-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return "", err } @@ -36,7 +36,7 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI // Terraform workspace can be overridden per component using `metadata.terraform_workspace_pattern` or `metadata.terraform_workspace_template` or `metadata.terraform_workspace` if terraformWorkspaceTemplate, terraformWorkspaceTemplateExist := componentMetadata["terraform_workspace_template"].(string); terraformWorkspaceTemplateExist { - tmpl, err = u.ProcessTmpl("terraform-workspace-template", terraformWorkspaceTemplate, configAndStacksInfo.ComponentSection, false) + tmpl, err = ProcessTmpl("terraform-workspace-template", terraformWorkspaceTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return "", err } diff --git a/internal/exec/template_funcs.go b/internal/exec/template_funcs.go new file mode 100644 index 000000000..118098e06 --- /dev/null +++ b/internal/exec/template_funcs.go @@ -0,0 +1,30 @@ +// https://forum.golangbridge.org/t/html-template-optional-argument-in-function/6080 +// https://lkumarjain.blogspot.com/2020/11/deep-dive-into-go-template.html +// https://echorand.me/posts/golang-templates/ +// https://www.practical-go-lessons.com/chap-32-templates +// https://docs.gofiber.io/template/next/html/TEMPLATES_CHEATSHEET/ +// https://engineering.01cloud.com/2023/04/13/optional-function-parameter-pattern/ + +package exec + +import ( + "context" + "text/template" +) + +// FuncMap creates and returns a map of template functions +func FuncMap(ctx context.Context) template.FuncMap { + atmosFuncs := &AtmosFuncs{ctx} + + return map[string]any{ + "atmos": func() any { return atmosFuncs }, + } +} + +type AtmosFuncs struct { + ctx context.Context +} + +func (AtmosFuncs) Component(component string, stack string) (any, error) { + return componentFunc(component, stack) +} diff --git a/internal/exec/template_funcs_component.go b/internal/exec/template_funcs_component.go new file mode 100644 index 000000000..4e5dc942b --- /dev/null +++ b/internal/exec/template_funcs_component.go @@ -0,0 +1,81 @@ +package exec + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-exec/tfexec" + "github.com/samber/lo" + + "github.com/cloudposse/atmos/pkg/utils" +) + +func componentFunc(component string, stack string) (any, error) { + sections, err := ExecuteDescribeComponent(component, stack) + if err != nil { + return nil, err + } + + executable, ok := sections["command"].(string) + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' does not have 'command' (executable) defined", component, stack) + } + + terraformWorkspace, ok := sections["workspace"].(string) + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' does not have Terraform/OpenTofu workspace defined", component, stack) + } + + componentInfo, ok := sections["component_info"] + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' does not have 'component_info' defined", component, stack) + } + + componentInfoMap, ok := componentInfo.(map[string]any) + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'component_info' section", component, stack) + } + + componentPath, ok := componentInfoMap["component_path"].(string) + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'component_info.component_path' section", component, stack) + } + + tf, err := tfexec.NewTerraform(componentPath, executable) + if err != nil { + return nil, err + } + + ctx := context.Background() + + err = tf.Init(ctx, tfexec.Upgrade(false)) + if err != nil { + return nil, err + } + + err = tf.WorkspaceNew(ctx, terraformWorkspace) + if err != nil { + err = tf.WorkspaceSelect(ctx, terraformWorkspace) + if err != nil { + return nil, err + } + } + + outputMeta, err := tf.Output(ctx) + if err != nil { + return nil, err + } + + outputMetaProcessed := lo.MapEntries(outputMeta, func(k string, v tfexec.OutputMeta) (string, any) { + d, _ := utils.ConvertFromJSON(string(v.Value)) + return k, d + }) + + outputs := map[string]any{ + "outputs": outputMetaProcessed, + } + + sections = lo.Assign(sections, outputs) + + return sections, nil +} diff --git a/pkg/utils/template_utils.go b/internal/exec/template_utils.go similarity index 96% rename from pkg/utils/template_utils.go rename to internal/exec/template_utils.go index 4607b9f50..538c2157d 100644 --- a/pkg/utils/template_utils.go +++ b/internal/exec/template_utils.go @@ -1,4 +1,4 @@ -package utils +package exec import ( "bytes" @@ -22,7 +22,9 @@ import ( // ProcessTmpl parses and executes Go templates func ProcessTmpl(tmplName string, tmplValue string, tmplData any, ignoreMissingTemplateValues bool) (string, error) { - t, err := template.New(tmplName).Funcs(sprig.FuncMap()).Parse(tmplValue) + funcs := lo.Assign(FuncMap(context.TODO()), sprig.FuncMap()) + + t, err := template.New(tmplName).Funcs(funcs).Parse(tmplValue) if err != nil { return "", err } @@ -87,7 +89,7 @@ func ProcessTmplWithDatasources( return "", err } - // Add Gomplate and Sprig functions and datasources + // Add Atmos, Gomplate and Sprig functions and datasources funcs := make(map[string]any) // Number of processing evaluations/passes @@ -126,6 +128,9 @@ func ProcessTmplWithDatasources( funcs = lo.Assign(funcs, sprig.FuncMap()) } + // Atmos functions + funcs = lo.Assign(funcs, FuncMap(context.TODO())) + // Process and add environment variables for k, v := range templateSettings.Env { err = os.Setenv(k, v) diff --git a/internal/exec/utils.go b/internal/exec/utils.go index 45bd3999e..8df6bca2f 100644 --- a/internal/exec/utils.go +++ b/internal/exec/utils.go @@ -13,7 +13,6 @@ import ( cfg "github.com/cloudposse/atmos/pkg/config" c "github.com/cloudposse/atmos/pkg/convert" "github.com/cloudposse/atmos/pkg/schema" - s "github.com/cloudposse/atmos/pkg/stack" u "github.com/cloudposse/atmos/pkg/utils" ) @@ -244,7 +243,7 @@ func FindStacksMap(cliConfig schema.CliConfiguration, ignoreMissingFiles bool) ( error, ) { // Process stack config file(s) - _, stacksMap, rawStackConfigs, err := s.ProcessYAMLConfigFiles( + _, stacksMap, rawStackConfigs, err := ProcessYAMLConfigFiles( cliConfig, cliConfig.StacksBaseAbsolutePath, cliConfig.TerraformDirAbsolutePath, @@ -350,7 +349,7 @@ func ProcessStacks( } if cliConfig.Stacks.NameTemplate != "" { - tmpl, err2 := u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + tmpl, err2 := ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err2 != nil { continue } @@ -426,7 +425,9 @@ func ProcessStacks( // Add Atmos component and stack configAndStacksInfo.ComponentSection["atmos_component"] = configAndStacksInfo.ComponentFromArg configAndStacksInfo.ComponentSection["atmos_stack"] = configAndStacksInfo.StackFromArg + configAndStacksInfo.ComponentSection["stack"] = configAndStacksInfo.StackFromArg configAndStacksInfo.ComponentSection["atmos_stack_file"] = configAndStacksInfo.StackFile + configAndStacksInfo.ComponentSection["atmos_manifest"] = configAndStacksInfo.StackFile // Add Atmos CLI config atmosCliConfig := map[string]any{} @@ -479,7 +480,14 @@ func ProcessStacks( return configAndStacksInfo, err } - componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "all-atmos-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) + componentSectionProcessed, err := ProcessTmplWithDatasources( + cliConfig, + settingsSectionStruct, + "all-atmos-sections", + componentSectionStr, + configAndStacksInfo.ComponentSection, + true, + ) if err != nil { // If any error returned from the templates processing, log it and exit u.LogErrorAndExit(err) diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go index 20e9f9275..ab231d6b6 100644 --- a/internal/exec/validate_stacks.go +++ b/internal/exec/validate_stacks.go @@ -11,7 +11,6 @@ import ( cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" - s "github.com/cloudposse/atmos/pkg/stack" u "github.com/cloudposse/atmos/pkg/utils" ) @@ -114,7 +113,7 @@ func ValidateStacks(cliConfig schema.CliConfiguration) error { path.Join(cliConfig.BasePath, cliConfig.Stacks.BasePath))) for _, filePath := range stackConfigFilesAbsolutePaths { - stackConfig, importsConfig, _, err := s.ProcessYAMLConfigFile( + stackConfig, importsConfig, _, err := ProcessYAMLConfigFile( cliConfig, cliConfig.StacksBaseAbsolutePath, filePath, @@ -133,7 +132,7 @@ func ValidateStacks(cliConfig schema.CliConfiguration) error { } // Process and validate the stack manifest - _, err = s.ProcessStackConfig( + _, err = ProcessStackConfig( cliConfig, cliConfig.StacksBaseAbsolutePath, cliConfig.TerraformDirAbsolutePath, @@ -244,7 +243,7 @@ func createComponentStackMap( // Find Atmos stack name if cliConfig.Stacks.NameTemplate != "" { - stackName, err = u.ProcessTmpl("validate-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + stackName, err = ProcessTmpl("validate-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -289,7 +288,9 @@ func checkComponentStackMap(componentStackMap map[string]map[string][]string) ([ // Hide the sections that should not be compared componentConfig["atmos_cli_config"] = nil componentConfig["atmos_stack"] = nil + componentConfig["stack"] = nil componentConfig["atmos_stack_file"] = nil + componentConfig["atmos_manifest"] = nil componentConfig["sources"] = nil componentConfig["imports"] = nil componentConfig["deps_all"] = nil diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index b54bf6d60..36d132317 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -284,7 +284,7 @@ func ExecuteAtmosVendorInternal( // Parse 'source' template if s.Version != "" { - uri, err = u.ProcessTmpl(fmt.Sprintf("source-%d-%s", indexSource, s.Version), s.Source, s, false) + uri, err = ProcessTmpl(fmt.Sprintf("source-%d-%s", indexSource, s.Version), s.Source, s, false) if err != nil { return err } @@ -318,7 +318,7 @@ func ExecuteAtmosVendorInternal( var target string // Parse 'target' template if s.Version != "" { - target, err = u.ProcessTmpl(fmt.Sprintf("target-%d-%d-%s", indexSource, indexTarget, s.Version), tgt, s, false) + target, err = ProcessTmpl(fmt.Sprintf("target-%d-%d-%s", indexSource, indexTarget, s.Version), tgt, s, false) if err != nil { return err } diff --git a/internal/exec/worflow.go b/internal/exec/worflow.go index a922784e8..ddfffeca2 100644 --- a/internal/exec/worflow.go +++ b/internal/exec/worflow.go @@ -7,11 +7,9 @@ import ( "path" "path/filepath" - "github.com/fatih/color" "github.com/spf13/cobra" "gopkg.in/yaml.v2" - tui "github.com/cloudposse/atmos/internal/tui/workflow" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" @@ -37,7 +35,7 @@ func ExecuteWorkflowCmd(cmd *cobra.Command, args []string) error { // If the `workflow` argument is not passed, start the workflow UI if len(args) != 1 { - workflowFile, workflow, fromStep, err = executeWorkflowUI(cliConfig) + workflowFile, workflow, fromStep, err = ExecuteWorkflowUI(cliConfig) if err != nil { return err } @@ -129,35 +127,3 @@ func ExecuteWorkflowCmd(cmd *cobra.Command, args []string) error { return nil } - -func executeWorkflowUI(cliConfig schema.CliConfiguration) (string, string, string, error) { - _, _, allWorkflows, err := ExecuteDescribeWorkflows(cliConfig) - if err != nil { - return "", "", "", err - } - - // Start the UI - app, err := tui.Execute(allWorkflows) - fmt.Println() - if err != nil { - return "", "", "", err - } - - selectedWorkflowFile := app.GetSelectedWorkflowFile() - selectedWorkflow := app.GetSelectedWorkflow() - selectedWorkflowStep := app.GetSelectedWorkflowStep() - - // If the user quit the UI, exit - if app.ExitStatusQuit() || selectedWorkflowFile == "" || selectedWorkflow == "" { - return "", "", "", nil - } - - fmt.Println() - u.PrintMessageInColor(fmt.Sprintf( - "Executing command:\n"+os.Args[0]+" workflow %s --file %s --from-step \"%s\"\n", selectedWorkflow, selectedWorkflowFile, selectedWorkflowStep), - color.New(color.FgCyan), - ) - fmt.Println() - - return selectedWorkflowFile, selectedWorkflow, selectedWorkflowStep, nil -} diff --git a/internal/exec/worflow_utils.go b/internal/exec/worflow_utils.go index bebb81df7..6809ed68f 100644 --- a/internal/exec/worflow_utils.go +++ b/internal/exec/worflow_utils.go @@ -7,10 +7,12 @@ import ( "sort" "strings" + "github.com/fatih/color" "github.com/pkg/errors" "github.com/samber/lo" "gopkg.in/yaml.v2" + w "github.com/cloudposse/atmos/internal/tui/workflow" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" ) @@ -204,3 +206,35 @@ func checkAndGenerateWorkflowStepNames(workflowDefinition *schema.WorkflowDefini } } } + +func ExecuteWorkflowUI(cliConfig schema.CliConfiguration) (string, string, string, error) { + _, _, allWorkflows, err := ExecuteDescribeWorkflows(cliConfig) + if err != nil { + return "", "", "", err + } + + // Start the UI + app, err := w.Execute(allWorkflows) + fmt.Println() + if err != nil { + return "", "", "", err + } + + selectedWorkflowFile := app.GetSelectedWorkflowFile() + selectedWorkflow := app.GetSelectedWorkflow() + selectedWorkflowStep := app.GetSelectedWorkflowStep() + + // If the user quit the UI, exit + if app.ExitStatusQuit() || selectedWorkflowFile == "" || selectedWorkflow == "" { + return "", "", "", nil + } + + fmt.Println() + u.PrintMessageInColor(fmt.Sprintf( + "Executing command:\n"+os.Args[0]+" workflow %s --file %s --from-step \"%s\"\n", selectedWorkflow, selectedWorkflowFile, selectedWorkflowStep), + color.New(color.FgCyan), + ) + fmt.Println() + + return selectedWorkflowFile, selectedWorkflow, selectedWorkflowStep, nil +} diff --git a/internal/tui/workflow/model.go b/internal/tui/workflow/model.go index d7f2cb211..ab9a50202 100644 --- a/internal/tui/workflow/model.go +++ b/internal/tui/workflow/model.go @@ -10,9 +10,9 @@ import ( "github.com/charmbracelet/lipgloss" mouseZone "github.com/lrstanley/bubblezone" "github.com/samber/lo" + "gopkg.in/yaml.v2" "github.com/cloudposse/atmos/pkg/schema" - u "github.com/cloudposse/atmos/pkg/utils" ) type App struct { @@ -212,7 +212,7 @@ func (app *App) initViews(workflows map[string]schema.WorkflowManifest) { } }) selectedWorkflowName := workflowsMapKeys[0] - selectedWorkflowContent, _ = u.ConvertToYAML(workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName]) + selectedWorkflowContent, _ = convertToYAML(workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName]) selectedWorkflowDefinition := workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName] stepItems = lo.Map(selectedWorkflowDefinition.Steps, func(s schema.WorkflowStep, _ int) list.Item { @@ -283,7 +283,7 @@ func (app *App) updateWorkflowFilesAndWorkflowsViews() { app.columnViews[1].list.SetItems(workflowItems) selectedWorkflowName := workflowsMapKeys[0] - selectedWorkflowContent, _ := u.ConvertToYAML(app.workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName]) + selectedWorkflowContent, _ := convertToYAML(app.workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName]) app.columnViews[2].SetContent(selectedWorkflowContent, "yaml") selectedWorkflowDefinition := app.workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName] @@ -310,7 +310,7 @@ func (app *App) updateWorkflowFilesAndWorkflowsViews() { } selectedWorkflowName := selectedWorkflow.(listItem).item - selectedWorkflowContent, _ := u.ConvertToYAML(app.workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName]) + selectedWorkflowContent, _ := convertToYAML(app.workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName]) app.columnViews[2].SetContent(selectedWorkflowContent, "yaml") selectedWorkflowDefinition := app.workflows[selectedWorkflowFileName].Workflows[selectedWorkflowName] @@ -361,3 +361,11 @@ func (app *App) execute() { app.selectedWorkflowStep = "" } } + +func convertToYAML(data any) (string, error) { + y, err := yaml.Marshal(data) + if err != nil { + return "", err + } + return string(y), nil +} diff --git a/pkg/stack/stack_processor.go b/pkg/stack/stack_processor.go index 5e30c302b..6e242f5ff 100644 --- a/pkg/stack/stack_processor.go +++ b/pkg/stack/stack_processor.go @@ -1,28 +1,8 @@ package stack import ( - "encoding/json" - "fmt" - "github.com/pkg/errors" - "github.com/santhosh-tekuri/jsonschema/v5" - "gopkg.in/yaml.v2" - "os" - "path" - "path/filepath" - "sort" - "strings" - "sync" - - cfg "github.com/cloudposse/atmos/pkg/config" - c "github.com/cloudposse/atmos/pkg/convert" - m "github.com/cloudposse/atmos/pkg/merge" + "github.com/cloudposse/atmos/internal/exec" "github.com/cloudposse/atmos/pkg/schema" - u "github.com/cloudposse/atmos/pkg/utils" -) - -var ( - // Mutex to serialize updates of the result map of ProcessYAMLConfigFiles function - processYAMLConfigFilesLock = &sync.Mutex{} ) // ProcessYAMLConfigFiles takes a list of paths to stack manifests, processes and deep-merges all imports, @@ -42,111 +22,18 @@ func ProcessYAMLConfigFiles( map[string]map[string]any, error, ) { - - count := len(filePaths) - listResult := make([]string, count) - mapResult := map[string]any{} - rawStackConfigs := map[string]map[string]any{} - var errorResult error - var wg sync.WaitGroup - wg.Add(count) - - for i, filePath := range filePaths { - go func(i int, p string) { - defer wg.Done() - - stackBasePath := stacksBasePath - if len(stackBasePath) < 1 { - stackBasePath = path.Dir(p) - } - - stackFileName := strings.TrimSuffix( - strings.TrimSuffix( - u.TrimBasePathFromPath(stackBasePath+"/", p), - cfg.DefaultStackConfigFileExtension), - ".yml", - ) - - deepMergedStackConfig, importsConfig, stackConfig, err := ProcessYAMLConfigFile( - cliConfig, - stackBasePath, - p, - map[string]map[any]any{}, - nil, - ignoreMissingFiles, - false, - false, - false, - map[any]any{}, - map[any]any{}, - "", - ) - - if err != nil { - errorResult = err - return - } - - var imports []string - for k := range importsConfig { - imports = append(imports, k) - } - - uniqueImports := u.UniqueStrings(imports) - sort.Strings(uniqueImports) - - componentStackMap := map[string]map[string][]string{} - - finalConfig, err := ProcessStackConfig( - cliConfig, - stackBasePath, - terraformComponentsBasePath, - helmfileComponentsBasePath, - p, - deepMergedStackConfig, - processStackDeps, - processComponentDeps, - "", - componentStackMap, - importsConfig, - true) - if err != nil { - errorResult = err - return - } - - finalConfig["imports"] = uniqueImports - - yamlConfig, err := yaml.Marshal(finalConfig) - if err != nil { - errorResult = err - return - } - - processYAMLConfigFilesLock.Lock() - defer processYAMLConfigFilesLock.Unlock() - - listResult[i] = string(yamlConfig) - mapResult[stackFileName] = finalConfig - rawStackConfigs[stackFileName] = map[string]any{} - rawStackConfigs[stackFileName]["stack"] = stackConfig - rawStackConfigs[stackFileName]["imports"] = importsConfig - rawStackConfigs[stackFileName]["import_files"] = uniqueImports - }(i, filePath) - } - - wg.Wait() - - if errorResult != nil { - return nil, nil, nil, errorResult - } - - return listResult, mapResult, rawStackConfigs, nil + return exec.ProcessYAMLConfigFiles( + cliConfig, + stacksBasePath, + terraformComponentsBasePath, + helmfileComponentsBasePath, + filePaths, + processStackDeps, + processComponentDeps, + ignoreMissingFiles, + ) } -// ProcessYAMLConfigFile takes a path to a YAML stack manifest, -// recursively processes and deep-merges all imports, -// and returns the final stack config func ProcessYAMLConfigFile( cliConfig schema.CliConfiguration, basePath string, @@ -166,304 +53,20 @@ func ProcessYAMLConfigFile( map[any]any, error, ) { - - var stackConfigs []map[any]any - relativeFilePath := u.TrimBasePathFromPath(basePath+"/", filePath) - - globalTerraformSection := map[any]any{} - globalHelmfileSection := map[any]any{} - globalOverrides := map[any]any{} - terraformOverrides := map[any]any{} - helmfileOverrides := map[any]any{} - finalTerraformOverrides := map[any]any{} - finalHelmfileOverrides := map[any]any{} - - stackYamlConfig, err := getFileContent(filePath) - - // If the file does not exist (`err != nil`), and `ignoreMissingFiles = true`, don't return the error. - // - // `ignoreMissingFiles = true` is used when executing `atmos describe affected` command. - // If we add a new stack manifest with some component configurations to the current branch, then the new file will not be present in - // the remote branch (with which the current branch is compared), and Atmos would throw an error. - // - // `skipIfMissing` is used in Atmos imports (https://atmos.tools/core-concepts/stacks/imports). - // Set it to `true` to ignore the imported manifest if it does not exist, and don't throw an error. - // This is useful when generating Atmos manifests using other tools, but the imported files are not present yet at the generation time. - if err != nil { - if ignoreMissingFiles || skipIfMissing { - return map[any]any{}, map[string]map[any]any{}, map[any]any{}, nil - } else { - return nil, nil, nil, err - } - } - - stackManifestTemplatesProcessed := stackYamlConfig - stackManifestTemplatesErrorMessage := "" - - // Process `Go` templates in the imported stack manifest using the provided `context` - // https://atmos.tools/core-concepts/stacks/imports#go-templates-in-imports - if !skipTemplatesProcessingInImports && len(context) > 0 { - stackManifestTemplatesProcessed, err = u.ProcessTmpl(relativeFilePath, stackYamlConfig, context, ignoreMissingTemplateValues) - if err != nil { - if cliConfig.Logs.Level == u.LogLevelTrace || cliConfig.Logs.Level == u.LogLevelDebug { - stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) - } - e := fmt.Errorf("invalid stack manifest '%s'\n%v%s", relativeFilePath, err, stackManifestTemplatesErrorMessage) - return nil, nil, nil, e - } - } - - stackConfigMap, err := c.YAMLToMapOfInterfaces(stackManifestTemplatesProcessed) - if err != nil { - if cliConfig.Logs.Level == u.LogLevelTrace || cliConfig.Logs.Level == u.LogLevelDebug { - stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) - } - e := fmt.Errorf("invalid stack manifest '%s'\n%v%s", relativeFilePath, err, stackManifestTemplatesErrorMessage) - return nil, nil, nil, e - } - - // If the path to the Atmos manifest JSON Schema is provided, validate the stack manifest against it - if atmosManifestJsonSchemaFilePath != "" { - // Convert the data to JSON and back to Go map to prevent the error: - // jsonschema: invalid jsonType: map[interface {}]interface {} - dataJson, err := u.ConvertToJSONFast(stackConfigMap) - if err != nil { - return nil, nil, nil, err - } - - dataFromJson, err := u.ConvertFromJSON(dataJson) - if err != nil { - return nil, nil, nil, err - } - - compiler := jsonschema.NewCompiler() - - atmosManifestJsonSchemaValidationErrorFormat := "Atmos manifest JSON Schema validation error in the file '%s':\n%v" - - atmosManifestJsonSchemaFileReader, err := os.Open(atmosManifestJsonSchemaFilePath) - if err != nil { - return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) - } - - if err := compiler.AddResource(atmosManifestJsonSchemaFilePath, atmosManifestJsonSchemaFileReader); err != nil { - return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) - } - - compiler.Draft = jsonschema.Draft2020 - - compiledSchema, err := compiler.Compile(atmosManifestJsonSchemaFilePath) - if err != nil { - return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) - } - - if err = compiledSchema.Validate(dataFromJson); err != nil { - switch e := err.(type) { - case *jsonschema.ValidationError: - b, err2 := json.MarshalIndent(e.BasicOutput(), "", " ") - if err2 != nil { - return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err2) - } - return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, string(b)) - default: - return nil, nil, nil, errors.Errorf(atmosManifestJsonSchemaValidationErrorFormat, relativeFilePath, err) - } - } - } - - // Check if the `overrides` sections exist and if we need to process overrides for the components in this stack manifest and its imports - - // Global overrides - if i, ok := stackConfigMap[cfg.OverridesSectionName]; ok { - if globalOverrides, ok = i.(map[any]any); !ok { - return nil, nil, nil, fmt.Errorf("invalid 'overrides' section in the stack manifest '%s'", relativeFilePath) - } - } - - // Terraform overrides - if o, ok := stackConfigMap["terraform"]; ok { - if globalTerraformSection, ok = o.(map[any]any); !ok { - return nil, nil, nil, fmt.Errorf("invalid 'terraform' section in the stack manifest '%s'", relativeFilePath) - } - - if i, ok := globalTerraformSection[cfg.OverridesSectionName]; ok { - if terraformOverrides, ok = i.(map[any]any); !ok { - return nil, nil, nil, fmt.Errorf("invalid 'terraform.overrides' section in the stack manifest '%s'", relativeFilePath) - } - } - } - - finalTerraformOverrides, err = m.Merge( - cliConfig, - []map[any]any{globalOverrides, terraformOverrides, parentTerraformOverrides}, - ) - if err != nil { - return nil, nil, nil, err - } - - // Helmfile overrides - if o, ok := stackConfigMap["helmfile"]; ok { - if globalHelmfileSection, ok = o.(map[any]any); !ok { - return nil, nil, nil, fmt.Errorf("invalid 'helmfile' section in the stack manifest '%s'", relativeFilePath) - } - - if i, ok := globalHelmfileSection[cfg.OverridesSectionName]; ok { - if helmfileOverrides, ok = i.(map[any]any); !ok { - return nil, nil, nil, fmt.Errorf("invalid 'terraform.overrides' section in the stack manifest '%s'", relativeFilePath) - } - } - } - - finalHelmfileOverrides, err = m.Merge( + return exec.ProcessYAMLConfigFile( cliConfig, - []map[any]any{globalOverrides, helmfileOverrides, parentHelmfileOverrides}, + basePath, + filePath, + importsConfig, + context, + ignoreMissingFiles, + skipTemplatesProcessingInImports, + ignoreMissingTemplateValues, + skipIfMissing, + parentTerraformOverrides, + parentHelmfileOverrides, + atmosManifestJsonSchemaFilePath, ) - if err != nil { - return nil, nil, nil, err - } - - // Add the `overrides` section for all components in this manifest - if len(finalTerraformOverrides) > 0 || len(finalHelmfileOverrides) > 0 { - if componentsSection, ok := stackConfigMap["components"].(map[any]any); ok { - // Terraform - if len(finalTerraformOverrides) > 0 { - if terraformSection, ok := componentsSection["terraform"].(map[any]any); ok { - for _, compSection := range terraformSection { - if componentSection, ok := compSection.(map[any]any); ok { - componentSection["overrides"] = finalTerraformOverrides - } - } - } - } - - // Helmfile - if len(finalHelmfileOverrides) > 0 { - if helmfileSection, ok := componentsSection["helmfile"].(map[any]any); ok { - for _, compSection := range helmfileSection { - if componentSection, ok := compSection.(map[any]any); ok { - componentSection["overrides"] = finalHelmfileOverrides - } - } - } - } - } - } - - // Find and process all imports - importStructs, err := processImportSection(stackConfigMap, relativeFilePath) - if err != nil { - return nil, nil, nil, err - } - - for _, importStruct := range importStructs { - imp := importStruct.Path - - if imp == "" { - return nil, nil, nil, fmt.Errorf("invalid empty import in the manifest '%s'", relativeFilePath) - } - - // If the import file is specified without extension, use `.yaml` as default - impWithExt := imp - ext := filepath.Ext(imp) - if ext == "" { - ext = cfg.DefaultStackConfigFileExtension - impWithExt = imp + ext - } - - impWithExtPath := path.Join(basePath, impWithExt) - - if impWithExtPath == filePath { - errorMessage := fmt.Sprintf("invalid import in the manifest '%s'\nThe file imports itself in '%s'", - relativeFilePath, - imp) - return nil, nil, nil, errors.New(errorMessage) - } - - // Find all import matches in the glob - importMatches, err := u.GetGlobMatches(impWithExtPath) - if err != nil || len(importMatches) == 0 { - // Retry (b/c we are using `doublestar` library and it sometimes has issues reading many files in a Docker container) - // TODO: review `doublestar` library - - importMatches, err = u.GetGlobMatches(impWithExtPath) - if err != nil || len(importMatches) == 0 { - // The import was not found -> check if the import is a Go template; if not, return the error - isGolangTemplate, err2 := u.IsGolangTemplate(imp) - if err2 != nil { - return nil, nil, nil, err2 - } - - // If the import is not a Go template and SkipIfMissing is false, return the error - if !isGolangTemplate && !importStruct.SkipIfMissing { - if err != nil { - errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'\nError: %s", - imp, - relativeFilePath, - err, - ) - return nil, nil, nil, errors.New(errorMessage) - } else if importMatches == nil { - errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'", - imp, - relativeFilePath, - ) - return nil, nil, nil, errors.New(errorMessage) - } - } - } - } - - // Support `context` in hierarchical imports. - // Deep-merge the parent `context` with the current `context` and propagate the result to the entire chain of imports. - // The parent `context` takes precedence over the current (imported) `context` and will override items with the same keys. - // TODO: instead of calling the conversion functions, we need to switch to generics and update everything to support it - listOfMaps := []map[any]any{c.MapsOfStringsToMapsOfInterfaces(importStruct.Context), c.MapsOfStringsToMapsOfInterfaces(context)} - mergedContext, err := m.Merge(cliConfig, listOfMaps) - if err != nil { - return nil, nil, nil, err - } - - // Process the imports in the current manifest - for _, importFile := range importMatches { - yamlConfig, _, yamlConfigRaw, err := ProcessYAMLConfigFile( - cliConfig, - basePath, - importFile, - importsConfig, - c.MapsOfInterfacesToMapsOfStrings(mergedContext), - ignoreMissingFiles, - importStruct.SkipTemplatesProcessing, - true, // importStruct.IgnoreMissingTemplateValues, - importStruct.SkipIfMissing, - finalTerraformOverrides, - finalHelmfileOverrides, - "", - ) - if err != nil { - return nil, nil, nil, err - } - - stackConfigs = append(stackConfigs, yamlConfig) - importRelativePathWithExt := strings.Replace(importFile, basePath+"/", "", 1) - ext2 := filepath.Ext(importRelativePathWithExt) - if ext2 == "" { - ext2 = cfg.DefaultStackConfigFileExtension - } - importRelativePathWithoutExt := strings.TrimSuffix(importRelativePathWithExt, ext2) - importsConfig[importRelativePathWithoutExt] = yamlConfigRaw - } - } - - if len(stackConfigMap) > 0 { - stackConfigs = append(stackConfigs, stackConfigMap) - } - - // Deep-merge the stack manifest and all the imports - stackConfigsDeepMerged, err := m.Merge(cliConfig, stackConfigs) - if err != nil { - return nil, nil, nil, err - } - - return stackConfigsDeepMerged, importsConfig, stackConfigMap, nil } // ProcessStackConfig takes a stack manifest, deep-merges all variables, settings, environments and backends, @@ -482,1016 +85,18 @@ func ProcessStackConfig( importsConfig map[string]map[any]any, checkBaseComponentExists bool, ) (map[any]any, error) { - - stackName := strings.TrimSuffix( - strings.TrimSuffix( - u.TrimBasePathFromPath(stacksBasePath+"/", stack), - cfg.DefaultStackConfigFileExtension), - ".yml", + return exec.ProcessStackConfig( + cliConfig, + stacksBasePath, + terraformComponentsBasePath, + helmfileComponentsBasePath, + stack, + config, + processStackDeps, + processComponentDeps, + componentTypeFilter, + componentStackMap, + importsConfig, + checkBaseComponentExists, ) - - globalVarsSection := map[any]any{} - globalSettingsSection := map[any]any{} - globalEnvSection := map[any]any{} - globalTerraformSection := map[any]any{} - globalHelmfileSection := map[any]any{} - globalComponentsSection := map[any]any{} - - terraformVars := map[any]any{} - terraformSettings := map[any]any{} - terraformEnv := map[any]any{} - terraformCommand := "" - terraformProviders := map[any]any{} - - helmfileVars := map[any]any{} - helmfileSettings := map[any]any{} - helmfileEnv := map[any]any{} - helmfileCommand := "" - - terraformComponents := map[string]any{} - helmfileComponents := map[string]any{} - allComponents := map[string]any{} - - // Global sections - if i, ok := config["vars"]; ok { - globalVarsSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'vars' section in the file '%s'", stackName) - } - } - - if i, ok := config["settings"]; ok { - globalSettingsSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'settings' section in the file '%s'", stackName) - } - } - - if i, ok := config["env"]; ok { - globalEnvSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'env' section in the file '%s'", stackName) - } - } - - if i, ok := config["terraform"]; ok { - globalTerraformSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform' section in the file '%s'", stackName) - } - } - - if i, ok := config["helmfile"]; ok { - globalHelmfileSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'helmfile' section in the file '%s'", stackName) - } - } - - if i, ok := config["components"]; ok { - globalComponentsSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components' section in the file '%s'", stackName) - } - } - - // Terraform section - if i, ok := globalTerraformSection[cfg.CommandSectionName]; ok { - terraformCommand, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.command' section in the file '%s'", stackName) - } - } - - if i, ok := globalTerraformSection["vars"]; ok { - terraformVars, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.vars' section in the file '%s'", stackName) - } - } - - globalAndTerraformVars, err := m.Merge(cliConfig, []map[any]any{globalVarsSection, terraformVars}) - if err != nil { - return nil, err - } - - if i, ok := globalTerraformSection["settings"]; ok { - terraformSettings, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.settings' section in the file '%s'", stackName) - } - } - - globalAndTerraformSettings, err := m.Merge(cliConfig, []map[any]any{globalSettingsSection, terraformSettings}) - if err != nil { - return nil, err - } - - if i, ok := globalTerraformSection["env"]; ok { - terraformEnv, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.env' section in the file '%s'", stackName) - } - } - - globalAndTerraformEnv, err := m.Merge(cliConfig, []map[any]any{globalEnvSection, terraformEnv}) - if err != nil { - return nil, err - } - - if i, ok := globalTerraformSection[cfg.ProvidersSectionName]; ok { - terraformProviders, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.providers' section in the file '%s'", stackName) - } - } - - // Global backend - globalBackendType := "" - globalBackendSection := map[any]any{} - - if i, ok := globalTerraformSection["backend_type"]; ok { - globalBackendType, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.backend_type' section in the file '%s'", stackName) - } - } - - if i, ok := globalTerraformSection["backend"]; ok { - globalBackendSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.backend' section in the file '%s'", stackName) - } - } - - // Global remote state backend - globalRemoteStateBackendType := "" - globalRemoteStateBackendSection := map[any]any{} - - if i, ok := globalTerraformSection["remote_state_backend_type"]; ok { - globalRemoteStateBackendType, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.remote_state_backend_type' section in the file '%s'", stackName) - } - } - - if i, ok := globalTerraformSection["remote_state_backend"]; ok { - globalRemoteStateBackendSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.remote_state_backend' section in the file '%s'", stackName) - } - } - - // Helmfile section - if i, ok := globalHelmfileSection[cfg.CommandSectionName]; ok { - helmfileCommand, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'helmfile.command' section in the file '%s'", stackName) - } - } - - if i, ok := globalHelmfileSection["vars"]; ok { - helmfileVars, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'helmfile.vars' section in the file '%s'", stackName) - } - } - - globalAndHelmfileVars, err := m.Merge(cliConfig, []map[any]any{globalVarsSection, helmfileVars}) - if err != nil { - return nil, err - } - - if i, ok := globalHelmfileSection["settings"]; ok { - helmfileSettings, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'helmfile.settings' section in the file '%s'", stackName) - } - } - - globalAndHelmfileSettings, err := m.Merge(cliConfig, []map[any]any{globalSettingsSection, helmfileSettings}) - if err != nil { - return nil, err - } - - if i, ok := globalHelmfileSection["env"]; ok { - helmfileEnv, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'helmfile.env' section in the file '%s'", stackName) - } - } - - globalAndHelmfileEnv, err := m.Merge(cliConfig, []map[any]any{globalEnvSection, helmfileEnv}) - if err != nil { - return nil, err - } - - // Process all Terraform components - if componentTypeFilter == "" || componentTypeFilter == "terraform" { - if allTerraformComponents, ok := globalComponentsSection["terraform"]; ok { - - allTerraformComponentsMap, ok := allTerraformComponents.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform' section in the file '%s'", stackName) - } - - for cmp, v := range allTerraformComponentsMap { - component := cmp.(string) - - componentMap, ok := v.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s' section in the file '%s'", component, stackName) - } - - componentVars := map[any]any{} - if i, ok := componentMap[cfg.VarsSectionName]; ok { - componentVars, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.vars' section in the file '%s'", component, stackName) - } - } - - componentSettings := map[any]any{} - if i, ok := componentMap[cfg.SettingsSectionName]; ok { - componentSettings, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.settings' section in the file '%s'", component, stackName) - } - - if i, ok := componentSettings["spacelift"]; ok { - _, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.settings.spacelift' section in the file '%s'", component, stackName) - } - } - } - - componentEnv := map[any]any{} - if i, ok := componentMap[cfg.EnvSectionName]; ok { - componentEnv, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.env' section in the file '%s'", component, stackName) - } - } - - componentProviders := map[any]any{} - if i, ok := componentMap[cfg.ProvidersSectionName]; ok { - componentProviders, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.providers' section in the file '%s'", component, stackName) - } - } - - // Component metadata. - // This is per component, not deep-merged and not inherited from base components and globals. - componentMetadata := map[any]any{} - if i, ok := componentMap[cfg.MetadataSectionName]; ok { - componentMetadata, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.metadata' section in the file '%s'", component, stackName) - } - } - - // Component backend - componentBackendType := "" - componentBackendSection := map[any]any{} - - if i, ok := componentMap[cfg.BackendTypeSectionName]; ok { - componentBackendType, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.backend_type' attribute in the file '%s'", component, stackName) - } - } - - if i, ok := componentMap[cfg.BackendSectionName]; ok { - componentBackendSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.backend' section in the file '%s'", component, stackName) - } - } - - // Component remote state backend - componentRemoteStateBackendType := "" - componentRemoteStateBackendSection := map[any]any{} - - if i, ok := componentMap["remote_state_backend_type"]; ok { - componentRemoteStateBackendType, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.remote_state_backend_type' attribute in the file '%s'", component, stackName) - } - } - - if i, ok := componentMap["remote_state_backend"]; ok { - componentRemoteStateBackendSection, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.remote_state_backend' section in the file '%s'", component, stackName) - } - } - - componentTerraformCommand := "" - if i, ok := componentMap[cfg.CommandSectionName]; ok { - componentTerraformCommand, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.command' attribute in the file '%s'", component, stackName) - } - } - - // Process overrides - componentOverrides := map[any]any{} - componentOverridesVars := map[any]any{} - componentOverridesSettings := map[any]any{} - componentOverridesEnv := map[any]any{} - componentOverridesProviders := map[any]any{} - componentOverridesTerraformCommand := "" - - if i, ok := componentMap[cfg.OverridesSectionName]; ok { - if componentOverrides, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides' in the manifest '%s'", component, stackName) - } - - if i, ok = componentOverrides[cfg.VarsSectionName]; ok { - if componentOverridesVars, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.vars' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.SettingsSectionName]; ok { - if componentOverridesSettings, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.settings' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.EnvSectionName]; ok { - if componentOverridesEnv, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.env' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.CommandSectionName]; ok { - if componentOverridesTerraformCommand, ok = i.(string); !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.command' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.ProvidersSectionName]; ok { - if componentOverridesProviders, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.overrides.providers' in the manifest '%s'", component, stackName) - } - } - } - - // Process base component(s) - baseComponentName := "" - baseComponentVars := map[any]any{} - baseComponentSettings := map[any]any{} - baseComponentEnv := map[any]any{} - baseComponentProviders := map[any]any{} - baseComponentTerraformCommand := "" - baseComponentBackendType := "" - baseComponentBackendSection := map[any]any{} - baseComponentRemoteStateBackendType := "" - baseComponentRemoteStateBackendSection := map[any]any{} - var baseComponentConfig schema.BaseComponentConfig - var componentInheritanceChain []string - var baseComponents []string - - // Inheritance using the top-level `component` attribute - if baseComponent, baseComponentExist := componentMap[cfg.ComponentSectionName]; baseComponentExist { - baseComponentName, ok = baseComponent.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.component' attribute in the file '%s'", component, stackName) - } - - // Process the base components recursively to find `componentInheritanceChain` - err = ProcessBaseComponentConfig( - cliConfig, - &baseComponentConfig, - allTerraformComponentsMap, - component, - stack, - baseComponentName, - terraformComponentsBasePath, - checkBaseComponentExists, - &baseComponents, - ) - if err != nil { - return nil, err - } - - baseComponentVars = baseComponentConfig.BaseComponentVars - baseComponentSettings = baseComponentConfig.BaseComponentSettings - baseComponentEnv = baseComponentConfig.BaseComponentEnv - baseComponentProviders = baseComponentConfig.BaseComponentProviders - baseComponentName = baseComponentConfig.FinalBaseComponentName - baseComponentTerraformCommand = baseComponentConfig.BaseComponentCommand - baseComponentBackendType = baseComponentConfig.BaseComponentBackendType - baseComponentBackendSection = baseComponentConfig.BaseComponentBackendSection - baseComponentRemoteStateBackendType = baseComponentConfig.BaseComponentRemoteStateBackendType - baseComponentRemoteStateBackendSection = baseComponentConfig.BaseComponentRemoteStateBackendSection - componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain - } - - // Multiple inheritance (and multiple-inheritance chain) using `metadata.component` and `metadata.inherit`. - // `metadata.component` points to the component implementation (e.g. in `components/terraform` folder), - // it does not specify inheritance (it overrides the deprecated top-level `component` attribute). - // `metadata.inherit` is a list of component names from which the current component inherits. - // It uses a method similar to Method Resolution Order (MRO), which is how Python supports multiple inheritance. - // - // In the case of multiple base components, it is processed left to right, in the order by which it was declared. - // For example: `metadata.inherits: [componentA, componentB]` - // will deep-merge all the base components of `componentA` (each component overriding its base), - // then all the base components of `componentB` (each component overriding its base), - // then the two results are deep-merged together (`componentB` inheritance chain will override values from 'componentA' inheritance chain). - if baseComponentFromMetadata, baseComponentFromMetadataExist := componentMetadata[cfg.ComponentSectionName]; baseComponentFromMetadataExist { - baseComponentName, ok = baseComponentFromMetadata.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.metadata.component' attribute in the file '%s'", component, stackName) - } - } - - baseComponents = append(baseComponents, baseComponentName) - - if inheritList, inheritListExist := componentMetadata["inherits"].([]any); inheritListExist { - for _, v := range inheritList { - baseComponentFromInheritList, ok := v.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.metadata.inherits' section in the file '%s'", component, stackName) - } - - if _, ok := allTerraformComponentsMap[baseComponentFromInheritList]; !ok { - if checkBaseComponentExists { - errorMessage := fmt.Sprintf("The component '%[1]s' in the stack manifest '%[2]s' inherits from '%[3]s' "+ - "(using 'metadata.inherits'), but '%[3]s' is not defined in any of the config files for the stack '%[2]s'", - component, - stackName, - baseComponentFromInheritList, - ) - return nil, errors.New(errorMessage) - } - } - - // Process the baseComponentFromInheritList components recursively to find `componentInheritanceChain` - err = ProcessBaseComponentConfig( - cliConfig, - &baseComponentConfig, - allTerraformComponentsMap, - component, - stack, - baseComponentFromInheritList, - terraformComponentsBasePath, - checkBaseComponentExists, - &baseComponents, - ) - if err != nil { - return nil, err - } - - baseComponentVars = baseComponentConfig.BaseComponentVars - baseComponentSettings = baseComponentConfig.BaseComponentSettings - baseComponentEnv = baseComponentConfig.BaseComponentEnv - baseComponentTerraformCommand = baseComponentConfig.BaseComponentCommand - baseComponentBackendType = baseComponentConfig.BaseComponentBackendType - baseComponentBackendSection = baseComponentConfig.BaseComponentBackendSection - baseComponentRemoteStateBackendType = baseComponentConfig.BaseComponentRemoteStateBackendType - baseComponentRemoteStateBackendSection = baseComponentConfig.BaseComponentRemoteStateBackendSection - componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain - } - } - - baseComponents = u.UniqueStrings(baseComponents) - sort.Strings(baseComponents) - - // Final configs - finalComponentVars, err := m.Merge( - cliConfig, - []map[any]any{ - globalAndTerraformVars, - baseComponentVars, - componentVars, - componentOverridesVars, - }) - if err != nil { - return nil, err - } - - finalComponentSettings, err := m.Merge( - cliConfig, - []map[any]any{ - globalAndTerraformSettings, - baseComponentSettings, - componentSettings, - componentOverridesSettings, - }) - if err != nil { - return nil, err - } - - finalComponentEnv, err := m.Merge( - cliConfig, - []map[any]any{ - globalAndTerraformEnv, - baseComponentEnv, - componentEnv, - componentOverridesEnv, - }) - if err != nil { - return nil, err - } - - finalComponentProviders, err := m.Merge( - cliConfig, - []map[any]any{ - terraformProviders, - baseComponentProviders, - componentProviders, - componentOverridesProviders, - }) - if err != nil { - return nil, err - } - - // Final backend - finalComponentBackendType := globalBackendType - if len(baseComponentBackendType) > 0 { - finalComponentBackendType = baseComponentBackendType - } - if len(componentBackendType) > 0 { - finalComponentBackendType = componentBackendType - } - - finalComponentBackendSection, err := m.Merge( - cliConfig, - []map[any]any{ - globalBackendSection, - baseComponentBackendSection, - componentBackendSection, - }) - if err != nil { - return nil, err - } - - finalComponentBackend := map[any]any{} - if i, ok := finalComponentBackendSection[finalComponentBackendType]; ok { - finalComponentBackend, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.backend' section for the component '%s'", component) - } - } - - // AWS S3 backend - // Check if `backend` section has `workspace_key_prefix` for `s3` backend type - // If it does not, use the component name instead - // It will also be propagated to `remote_state_backend` section of `s3` type - if finalComponentBackendType == "s3" { - if p, ok := finalComponentBackend["workspace_key_prefix"].(string); !ok || p == "" { - workspaceKeyPrefix := component - if baseComponentName != "" { - workspaceKeyPrefix = baseComponentName - } - finalComponentBackend["workspace_key_prefix"] = strings.Replace(workspaceKeyPrefix, "/", "-", -1) - } - } - - // Google GSC backend - // Check if `backend` section has `prefix` for `gcs` backend type - // If it does not, use the component name instead - // https://developer.hashicorp.com/terraform/language/settings/backends/gcs - // https://developer.hashicorp.com/terraform/language/settings/backends/gcs#prefix - if finalComponentBackendType == "gcs" { - if p, ok := finalComponentBackend["prefix"].(string); !ok || p == "" { - prefix := component - if baseComponentName != "" { - prefix = baseComponentName - } - finalComponentBackend["prefix"] = strings.Replace(prefix, "/", "-", -1) - } - } - - // Azure backend - // Check if component `backend` section has `key` for `azurerm` backend type - // If it does not, use the component name instead and format it with the global backend key name to auto generate a unique Terraform state key - // The backend state file will be formatted like so: {global key name}/{component name}.terraform.tfstate - if finalComponentBackendType == "azurerm" { - if componentAzurerm, componentAzurermExists := componentBackendSection["azurerm"].(map[any]any); !componentAzurermExists { - if _, componentAzurermKeyExists := componentAzurerm["key"].(string); !componentAzurermKeyExists { - azureKeyPrefixComponent := component - var keyName []string - if baseComponentName != "" { - azureKeyPrefixComponent = baseComponentName - } - if globalAzurerm, globalAzurermExists := globalBackendSection["azurerm"].(map[any]any); globalAzurermExists { - if _, globalAzurermKeyExists := globalAzurerm["key"].(string); globalAzurermKeyExists { - keyName = append(keyName, globalAzurerm["key"].(string)) - } - } - componentKeyName := strings.Replace(azureKeyPrefixComponent, "/", "-", -1) - keyName = append(keyName, fmt.Sprintf("%s.terraform.tfstate", componentKeyName)) - finalComponentBackend["key"] = strings.Join(keyName, "/") - } - } - } - - // Final remote state backend - finalComponentRemoteStateBackendType := finalComponentBackendType - if len(globalRemoteStateBackendType) > 0 { - finalComponentRemoteStateBackendType = globalRemoteStateBackendType - } - if len(baseComponentRemoteStateBackendType) > 0 { - finalComponentRemoteStateBackendType = baseComponentRemoteStateBackendType - } - if len(componentRemoteStateBackendType) > 0 { - finalComponentRemoteStateBackendType = componentRemoteStateBackendType - } - - finalComponentRemoteStateBackendSection, err := m.Merge( - cliConfig, - []map[any]any{ - globalRemoteStateBackendSection, - baseComponentRemoteStateBackendSection, - componentRemoteStateBackendSection, - }) - if err != nil { - return nil, err - } - - // Merge `backend` and `remote_state_backend` sections - // This will allow keeping `remote_state_backend` section DRY - finalComponentRemoteStateBackendSectionMerged, err := m.Merge( - cliConfig, - []map[any]any{ - finalComponentBackendSection, - finalComponentRemoteStateBackendSection, - }) - if err != nil { - return nil, err - } - - finalComponentRemoteStateBackend := map[any]any{} - if i, ok := finalComponentRemoteStateBackendSectionMerged[finalComponentRemoteStateBackendType]; ok { - finalComponentRemoteStateBackend, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'terraform.remote_state_backend' section for the component '%s'", component) - } - } - - // Final binary to execute - // Check for the binary in the following order: - // - `components.terraform.command` section in `atmos.yaml` CLI config file - // - global `terraform.command` section - // - base component(s) `command` section - // - component `command` section - // - `overrides.command` section - finalComponentTerraformCommand := "terraform" - if cliConfig.Components.Terraform.Command != "" { - finalComponentTerraformCommand = cliConfig.Components.Terraform.Command - } - if terraformCommand != "" { - finalComponentTerraformCommand = terraformCommand - } - if baseComponentTerraformCommand != "" { - finalComponentTerraformCommand = baseComponentTerraformCommand - } - if componentTerraformCommand != "" { - finalComponentTerraformCommand = componentTerraformCommand - } - if componentOverridesTerraformCommand != "" { - finalComponentTerraformCommand = componentOverridesTerraformCommand - } - - // If the component is not deployable (`metadata.type: abstract`), remove `settings.spacelift.workspace_enabled` from the map). - // This will prevent the derived components from inheriting `settings.spacelift.workspace_enabled=false` of not-deployable components. - // Also, removing `settings.spacelift.workspace_enabled` will effectively make it `false` - // and `spacelift_stack_processor` will not create a Spacelift stack for the abstract component - // even if `settings.spacelift.workspace_enabled` was set to `true`. - // This is per component, not deep-merged and not inherited from base components and globals. - componentIsAbstract := false - if componentType, componentTypeAttributeExists := componentMetadata["type"].(string); componentTypeAttributeExists { - if componentType == "abstract" { - componentIsAbstract = true - } - } - if componentIsAbstract { - if i, ok := finalComponentSettings["spacelift"]; ok { - spaceliftSettings, ok := i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.terraform.%s.settings.spacelift' section in the file '%s'", component, stackName) - } - delete(spaceliftSettings, "workspace_enabled") - } - } - - comp := map[string]any{} - comp[cfg.VarsSectionName] = finalComponentVars - comp[cfg.SettingsSectionName] = finalComponentSettings - comp[cfg.EnvSectionName] = finalComponentEnv - comp[cfg.BackendTypeSectionName] = finalComponentBackendType - comp[cfg.BackendSectionName] = finalComponentBackend - comp["remote_state_backend_type"] = finalComponentRemoteStateBackendType - comp["remote_state_backend"] = finalComponentRemoteStateBackend - comp[cfg.CommandSectionName] = finalComponentTerraformCommand - comp["inheritance"] = componentInheritanceChain - comp[cfg.MetadataSectionName] = componentMetadata - comp[cfg.OverridesSectionName] = componentOverrides - comp[cfg.ProvidersSectionName] = finalComponentProviders - - if baseComponentName != "" { - comp[cfg.ComponentSectionName] = baseComponentName - } - - terraformComponents[component] = comp - } - } - } - - // Process all helmfile components - if componentTypeFilter == "" || componentTypeFilter == "helmfile" { - if allHelmfileComponents, ok := globalComponentsSection["helmfile"]; ok { - - allHelmfileComponentsMap, ok := allHelmfileComponents.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile' section in the file '%s'", stackName) - } - - for cmp, v := range allHelmfileComponentsMap { - component := cmp.(string) - - componentMap, ok := v.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s' section in the file '%s'", component, stackName) - } - - componentVars := map[any]any{} - if i2, ok := componentMap[cfg.VarsSectionName]; ok { - componentVars, ok = i2.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.vars' section in the file '%s'", component, stackName) - } - } - - componentSettings := map[any]any{} - if i, ok := componentMap[cfg.SettingsSectionName]; ok { - componentSettings, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.settings' section in the file '%s'", component, stackName) - } - } - - componentEnv := map[any]any{} - if i, ok := componentMap[cfg.EnvSectionName]; ok { - componentEnv, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.env' section in the file '%s'", component, stackName) - } - } - - // Component metadata. - // This is per component, not deep-merged and not inherited from base components and globals. - componentMetadata := map[any]any{} - if i, ok := componentMap[cfg.MetadataSectionName]; ok { - componentMetadata, ok = i.(map[any]any) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.metadata' section in the file '%s'", component, stackName) - } - } - - componentHelmfileCommand := "" - if i, ok := componentMap[cfg.CommandSectionName]; ok { - componentHelmfileCommand, ok = i.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.command' attribute in the file '%s'", component, stackName) - } - } - - // Process overrides - componentOverrides := map[any]any{} - componentOverridesVars := map[any]any{} - componentOverridesSettings := map[any]any{} - componentOverridesEnv := map[any]any{} - componentOverridesHelmfileCommand := "" - - if i, ok := componentMap[cfg.OverridesSectionName]; ok { - if componentOverrides, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides' in the manifest '%s'", component, stackName) - } - - if i, ok = componentOverrides[cfg.VarsSectionName]; ok { - if componentOverridesVars, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.vars' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.SettingsSectionName]; ok { - if componentOverridesSettings, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.settings' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.EnvSectionName]; ok { - if componentOverridesEnv, ok = i.(map[any]any); !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.env' in the manifest '%s'", component, stackName) - } - } - - if i, ok = componentOverrides[cfg.CommandSectionName]; ok { - if componentOverridesHelmfileCommand, ok = i.(string); !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.overrides.command' in the manifest '%s'", component, stackName) - } - } - } - - // Process base component(s) - baseComponentVars := map[any]any{} - baseComponentSettings := map[any]any{} - baseComponentEnv := map[any]any{} - baseComponentName := "" - baseComponentHelmfileCommand := "" - var baseComponentConfig schema.BaseComponentConfig - var componentInheritanceChain []string - var baseComponents []string - - // Inheritance using the top-level `component` attribute - if baseComponent, baseComponentExist := componentMap[cfg.ComponentSectionName]; baseComponentExist { - baseComponentName, ok = baseComponent.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.component' attribute in the file '%s'", component, stackName) - } - - // Process the base components recursively to find `componentInheritanceChain` - err = ProcessBaseComponentConfig( - cliConfig, - &baseComponentConfig, - allHelmfileComponentsMap, - component, - stack, - baseComponentName, - helmfileComponentsBasePath, - checkBaseComponentExists, - &baseComponents, - ) - if err != nil { - return nil, err - } - - baseComponentVars = baseComponentConfig.BaseComponentVars - baseComponentSettings = baseComponentConfig.BaseComponentSettings - baseComponentEnv = baseComponentConfig.BaseComponentEnv - baseComponentName = baseComponentConfig.FinalBaseComponentName - baseComponentHelmfileCommand = baseComponentConfig.BaseComponentCommand - componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain - } - - // Multiple inheritance (and multiple-inheritance chain) using `metadata.component` and `metadata.inherit`. - // `metadata.component` points to the component implementation (e.g. in `components/terraform` folder), - // it does not specify inheritance (it overrides the deprecated top-level `component` attribute). - // `metadata.inherit` is a list of component names from which the current component inherits. - // It uses a method similar to Method Resolution Order (MRO), which is how Python supports multiple inheritance. - // - // In the case of multiple base components, it is processed left to right, in the order by which it was declared. - // For example: `metadata.inherits: [componentA, componentB]` - // will deep-merge all the base components of `componentA` (each component overriding its base), - // then all the base components of `componentB` (each component overriding its base), - // then the two results are deep-merged together (`componentB` inheritance chain will override values from 'componentA' inheritance chain). - if baseComponentFromMetadata, baseComponentFromMetadataExist := componentMetadata[cfg.ComponentSectionName]; baseComponentFromMetadataExist { - baseComponentName, ok = baseComponentFromMetadata.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.metadata.component' attribute in the file '%s'", component, stackName) - } - } - - baseComponents = append(baseComponents, baseComponentName) - - if inheritList, inheritListExist := componentMetadata["inherits"].([]any); inheritListExist { - for _, v := range inheritList { - baseComponentFromInheritList, ok := v.(string) - if !ok { - return nil, fmt.Errorf("invalid 'components.helmfile.%s.metadata.inherits' section in the file '%s'", component, stackName) - } - - if _, ok := allHelmfileComponentsMap[baseComponentFromInheritList]; !ok { - if checkBaseComponentExists { - errorMessage := fmt.Sprintf("The component '%[1]s' in the stack manifest '%[2]s' inherits from '%[3]s' "+ - "(using 'metadata.inherits'), but '%[3]s' is not defined in any of the config files for the stack '%[2]s'", - component, - stackName, - baseComponentFromInheritList, - ) - return nil, errors.New(errorMessage) - } - } - - // Process the baseComponentFromInheritList components recursively to find `componentInheritanceChain` - err = ProcessBaseComponentConfig( - cliConfig, - &baseComponentConfig, - allHelmfileComponentsMap, - component, - stack, - baseComponentFromInheritList, - helmfileComponentsBasePath, - checkBaseComponentExists, - &baseComponents, - ) - if err != nil { - return nil, err - } - - baseComponentVars = baseComponentConfig.BaseComponentVars - baseComponentSettings = baseComponentConfig.BaseComponentSettings - baseComponentEnv = baseComponentConfig.BaseComponentEnv - baseComponentName = baseComponentConfig.FinalBaseComponentName - baseComponentHelmfileCommand = baseComponentConfig.BaseComponentCommand - componentInheritanceChain = baseComponentConfig.ComponentInheritanceChain - } - } - - baseComponents = u.UniqueStrings(baseComponents) - sort.Strings(baseComponents) - - // Final configs - finalComponentVars, err := m.Merge( - cliConfig, - []map[any]any{ - globalAndHelmfileVars, - baseComponentVars, - componentVars, - componentOverridesVars, - }) - if err != nil { - return nil, err - } - - finalComponentSettings, err := m.Merge( - cliConfig, - []map[any]any{ - globalAndHelmfileSettings, - baseComponentSettings, - componentSettings, - componentOverridesSettings, - }) - if err != nil { - return nil, err - } - - finalComponentEnv, err := m.Merge( - cliConfig, - []map[any]any{ - globalAndHelmfileEnv, - baseComponentEnv, - componentEnv, - componentOverridesEnv, - }) - if err != nil { - return nil, err - } - - // Final binary to execute - // Check for the binary in the following order: - // - `components.helmfile.command` section in `atmos.yaml` CLI config file - // - global `helmfile.command` section - // - base component(s) `command` section - // - component `command` section - // - `overrides.command` section - finalComponentHelmfileCommand := "helmfile" - if cliConfig.Components.Helmfile.Command != "" { - finalComponentHelmfileCommand = cliConfig.Components.Helmfile.Command - } - if helmfileCommand != "" { - finalComponentHelmfileCommand = helmfileCommand - } - if baseComponentHelmfileCommand != "" { - finalComponentHelmfileCommand = baseComponentHelmfileCommand - } - if componentHelmfileCommand != "" { - finalComponentHelmfileCommand = componentHelmfileCommand - } - if componentOverridesHelmfileCommand != "" { - finalComponentHelmfileCommand = componentOverridesHelmfileCommand - } - - comp := map[string]any{} - comp[cfg.VarsSectionName] = finalComponentVars - comp[cfg.SettingsSectionName] = finalComponentSettings - comp[cfg.EnvSectionName] = finalComponentEnv - comp[cfg.CommandSectionName] = finalComponentHelmfileCommand - comp["inheritance"] = componentInheritanceChain - comp[cfg.MetadataSectionName] = componentMetadata - comp[cfg.OverridesSectionName] = componentOverrides - - if baseComponentName != "" { - comp[cfg.ComponentSectionName] = baseComponentName - } - - helmfileComponents[component] = comp - } - } - } - - allComponents["terraform"] = terraformComponents - allComponents["helmfile"] = helmfileComponents - - result := map[any]any{ - "components": allComponents, - } - - return result, nil } diff --git a/pkg/stack/stack_processor_utils.go b/pkg/stack/stack_processor_utils.go deleted file mode 100644 index b9b654fce..000000000 --- a/pkg/stack/stack_processor_utils.go +++ /dev/null @@ -1,647 +0,0 @@ -package stack - -import ( - "errors" - "fmt" - "os" - "path" - "path/filepath" - "reflect" - "sort" - "strings" - "sync" - - "github.com/mitchellh/mapstructure" - - cfg "github.com/cloudposse/atmos/pkg/config" - c "github.com/cloudposse/atmos/pkg/convert" - m "github.com/cloudposse/atmos/pkg/merge" - "github.com/cloudposse/atmos/pkg/schema" - u "github.com/cloudposse/atmos/pkg/utils" -) - -var ( - getFileContentSyncMap = sync.Map{} -) - -// FindComponentStacks finds all infrastructure stack manifests where the component or the base component is defined -func FindComponentStacks( - componentType string, - component string, - baseComponent string, - componentStackMap map[string]map[string][]string) ([]string, error) { - - var stacks []string - - if componentStackConfig, componentStackConfigExists := componentStackMap[componentType]; componentStackConfigExists { - if componentStacks, componentStacksExist := componentStackConfig[component]; componentStacksExist { - stacks = append(stacks, componentStacks...) - } - - if baseComponent != "" { - if baseComponentStacks, baseComponentStacksExist := componentStackConfig[baseComponent]; baseComponentStacksExist { - stacks = append(stacks, baseComponentStacks...) - } - } - } - - unique := u.UniqueStrings(stacks) - sort.Strings(unique) - return unique, nil -} - -// FindComponentDependenciesLegacy finds all imports where the component or the base component(s) are defined -// Component depends on the imported config file if any of the following conditions is true: -// 1. The imported config file has any of the global `backend`, `backend_type`, `env`, `remote_state_backend`, `remote_state_backend_type`, -// `settings` or `vars` sections which are not empty. -// 2. The imported config file has the component type section, which has any of the `backend`, `backend_type`, `env`, `remote_state_backend`, -// `remote_state_backend_type`, `settings` or `vars` sections which are not empty. -// 3. The imported config file has the "components" section, which has the component type section, which has the component section. -// 4. The imported config file has the "components" section, which has the component type section, which has the base component(s) section, -// and the base component section is defined inline (not imported). -func FindComponentDependenciesLegacy( - stack string, - componentType string, - component string, - baseComponents []string, - stackImports map[string]map[any]any) ([]string, error) { - - var deps []string - - sectionsToCheck := []string{ - "backend", - "backend_type", - "env", - "remote_state_backend", - "remote_state_backend_type", - "settings", - "vars", - } - - for stackImportName, stackImportMap := range stackImports { - - if sectionContainsAnyNotEmptySections(stackImportMap, sectionsToCheck) { - deps = append(deps, stackImportName) - continue - } - - if sectionContainsAnyNotEmptySections(stackImportMap, []string{componentType}) { - if sectionContainsAnyNotEmptySections(stackImportMap[componentType].(map[any]any), sectionsToCheck) { - deps = append(deps, stackImportName) - continue - } - } - - stackImportMapComponentsSection, ok := stackImportMap["components"].(map[any]any) - if !ok { - continue - } - - stackImportMapComponentTypeSection, ok := stackImportMapComponentsSection[componentType].(map[any]any) - if !ok { - continue - } - - if stackImportMapComponentSection, ok := stackImportMapComponentTypeSection[component].(map[any]any); ok { - if len(stackImportMapComponentSection) > 0 { - deps = append(deps, stackImportName) - continue - } - } - - // Process base component(s) - // Only include the imported config file into "deps" if all the following conditions are `true`: - // 1. The imported config file has the base component(s) section(s) - // 2. The imported config file does not import other config files (which means that instead it defined the base component sections inline) - // 3. If the imported config file does import other config files, check that the base component sections in them are different by using - // `reflect.DeepEqual`. If they are the same, don't include the imported config file since it does not specify anything for the base component - for _, baseComponent := range baseComponents { - baseComponentSection, ok := stackImportMapComponentTypeSection[baseComponent].(map[any]any) - - if !ok || len(baseComponentSection) == 0 { - continue - } - - importOfStackImportStructs, err := processImportSection(stackImportMap, stack) - if err != nil { - return nil, err - } - - if len(importOfStackImportStructs) == 0 { - deps = append(deps, stackImportName) - continue - } - - for _, importOfStackImportStruct := range importOfStackImportStructs { - importOfStackImportMap, ok := stackImports[importOfStackImportStruct.Path] - if !ok { - continue - } - - importOfStackImportComponentsSection, ok := importOfStackImportMap["components"].(map[any]any) - if !ok { - continue - } - - importOfStackImportComponentTypeSection, ok := importOfStackImportComponentsSection[componentType].(map[any]any) - if !ok { - continue - } - - importOfStackImportBaseComponentSection, ok := importOfStackImportComponentTypeSection[baseComponent].(map[any]any) - if !ok { - continue - } - - if !reflect.DeepEqual(baseComponentSection, importOfStackImportBaseComponentSection) { - deps = append(deps, stackImportName) - break - } - } - } - } - - deps = append(deps, stack) - unique := u.UniqueStrings(deps) - sort.Strings(unique) - return unique, nil -} - -// processImportSection processes the `import` section in stack manifests -// The `import` section` can be of the following types: -// 1. list of `StackImport` structs -// 2. list of strings -// 3. List of strings and `StackImport` structs in the same file -func processImportSection(stackMap map[any]any, filePath string) ([]schema.StackImport, error) { - stackImports, ok := stackMap[cfg.ImportSectionName] - - // If the stack file does not have the `import` section, return - if !ok || stackImports == nil { - return nil, nil - } - - // Check if the `import` section is a list of objects - importsList, ok := stackImports.([]any) - if !ok || len(importsList) == 0 { - return nil, fmt.Errorf("invalid 'import' section in the file '%s'", filePath) - } - - var result []schema.StackImport - - for _, imp := range importsList { - if imp == nil { - return nil, fmt.Errorf("invalid import in the file '%s'", filePath) - } - - // 1. Try to decode the import as the `StackImport` struct - var importObj schema.StackImport - err := mapstructure.Decode(imp, &importObj) - if err == nil { - result = append(result, importObj) - continue - } - - // 2. Try to cast the import to a string - s, ok := imp.(string) - if !ok { - return nil, fmt.Errorf("invalid import '%v' in the file '%s'", imp, filePath) - } - if s == "" { - return nil, fmt.Errorf("invalid empty import in the file '%s'", filePath) - } - - result = append(result, schema.StackImport{Path: s}) - } - - return result, nil -} - -// sectionContainsAnyNotEmptySections checks if a section contains any of the provided low-level sections, and it's not empty -func sectionContainsAnyNotEmptySections(section map[any]any, sectionsToCheck []string) bool { - for _, s := range sectionsToCheck { - if len(s) > 0 { - if v, ok := section[s]; ok { - if v2, ok2 := v.(map[any]any); ok2 && len(v2) > 0 { - return true - } - if v2, ok2 := v.(string); ok2 && len(v2) > 0 { - return true - } - } - } - } - return false -} - -// CreateComponentStackMap accepts a config file and creates a map of component-stack dependencies -func CreateComponentStackMap( - cliConfig schema.CliConfiguration, - stacksBasePath string, - terraformComponentsBasePath string, - helmfileComponentsBasePath string, - filePath string, -) (map[string]map[string][]string, error) { - - stackComponentMap := map[string]map[string][]string{} - stackComponentMap["terraform"] = map[string][]string{} - stackComponentMap["helmfile"] = map[string][]string{} - - componentStackMap := map[string]map[string][]string{} - componentStackMap["terraform"] = map[string][]string{} - componentStackMap["helmfile"] = map[string][]string{} - - dir := path.Dir(filePath) - - err := filepath.Walk(dir, - func(p string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - isDirectory, err := u.IsDirectory(p) - if err != nil { - return err - } - - isYaml := u.IsYaml(p) - - if !isDirectory && isYaml { - config, _, _, err := ProcessYAMLConfigFile( - cliConfig, - stacksBasePath, - p, - map[string]map[any]any{}, - nil, - false, - false, - false, - false, - map[any]any{}, - map[any]any{}, - "", - ) - if err != nil { - return err - } - - finalConfig, err := ProcessStackConfig( - cliConfig, - stacksBasePath, - terraformComponentsBasePath, - helmfileComponentsBasePath, - p, - config, - false, - false, - "", - nil, - nil, - true) - if err != nil { - return err - } - - if componentsConfig, componentsConfigExists := finalConfig["components"]; componentsConfigExists { - componentsSection := componentsConfig.(map[string]any) - stackName := strings.Replace(p, stacksBasePath+"/", "", 1) - - if terraformConfig, terraformConfigExists := componentsSection["terraform"]; terraformConfigExists { - terraformSection := terraformConfig.(map[string]any) - - for k := range terraformSection { - stackComponentMap["terraform"][stackName] = append(stackComponentMap["terraform"][stackName], k) - } - } - - if helmfileConfig, helmfileConfigExists := componentsSection["helmfile"]; helmfileConfigExists { - helmfileSection := helmfileConfig.(map[string]any) - - for k := range helmfileSection { - stackComponentMap["helmfile"][stackName] = append(stackComponentMap["helmfile"][stackName], k) - } - } - } - } - - return nil - }) - - if err != nil { - return nil, err - } - - for stack, components := range stackComponentMap["terraform"] { - for _, component := range components { - componentStackMap["terraform"][component] = append(componentStackMap["terraform"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1)) - } - } - - for stack, components := range stackComponentMap["helmfile"] { - for _, component := range components { - componentStackMap["helmfile"][component] = append(componentStackMap["helmfile"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1)) - } - } - - return componentStackMap, nil -} - -// getFileContent tries to read and return the file content from the sync map if it exists in the map, -// otherwise it reads the file, stores its content in the map and returns the content -func getFileContent(filePath string) (string, error) { - existingContent, found := getFileContentSyncMap.Load(filePath) - if found && existingContent != nil { - return fmt.Sprintf("%s", existingContent), nil - } - - content, err := os.ReadFile(filePath) - if err != nil { - return "", err - } - getFileContentSyncMap.Store(filePath, content) - - return string(content), nil -} - -// ProcessBaseComponentConfig processes base component(s) config -func ProcessBaseComponentConfig( - cliConfig schema.CliConfiguration, - baseComponentConfig *schema.BaseComponentConfig, - allComponentsMap map[any]any, - component string, - stack string, - baseComponent string, - componentBasePath string, - checkBaseComponentExists bool, - baseComponents *[]string, -) error { - - if component == baseComponent { - return nil - } - - var baseComponentVars map[any]any - var baseComponentSettings map[any]any - var baseComponentEnv map[any]any - var baseComponentProviders map[any]any - var baseComponentCommand string - var baseComponentBackendType string - var baseComponentBackendSection map[any]any - var baseComponentRemoteStateBackendType string - var baseComponentRemoteStateBackendSection map[any]any - var baseComponentMap map[any]any - var ok bool - - *baseComponents = append(*baseComponents, baseComponent) - - if baseComponentSection, baseComponentSectionExist := allComponentsMap[baseComponent]; baseComponentSectionExist { - baseComponentMap, ok = baseComponentSection.(map[any]any) - if !ok { - // Depending on the code and libraries, the section can have different map types: map[any]any or map[string]any - // We try to convert to both - baseComponentMapOfStrings, ok := baseComponentSection.(map[string]any) - if !ok { - return fmt.Errorf("invalid config for the base component '%s' of the component '%s' in the stack '%s'", - baseComponent, component, stack) - } - baseComponentMap = c.MapsOfStringsToMapsOfInterfaces(baseComponentMapOfStrings) - } - - // First, process the base component(s) of this base component - if baseComponentOfBaseComponent, baseComponentOfBaseComponentExist := baseComponentMap["component"]; baseComponentOfBaseComponentExist { - baseComponentOfBaseComponentString, ok := baseComponentOfBaseComponent.(string) - if !ok { - return fmt.Errorf("invalid 'component:' section of the component '%s' in the stack '%s'", - baseComponent, stack) - } - - err := ProcessBaseComponentConfig( - cliConfig, - baseComponentConfig, - allComponentsMap, - baseComponent, - stack, - baseComponentOfBaseComponentString, - componentBasePath, - checkBaseComponentExists, - baseComponents, - ) - - if err != nil { - return err - } - } - - // Base component metadata. - // This is per component, not deep-merged and not inherited from base components and globals. - componentMetadata := map[any]any{} - if i, ok := baseComponentMap["metadata"]; ok { - componentMetadata, ok = i.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.metadata' section in the stack '%s'", component, stack) - } - - if inheritList, inheritListExist := componentMetadata["inherits"].([]any); inheritListExist { - for _, v := range inheritList { - baseComponentFromInheritList, ok := v.(string) - if !ok { - return fmt.Errorf("invalid '%s.metadata.inherits' section in the stack '%s'", component, stack) - } - - if _, ok := allComponentsMap[baseComponentFromInheritList]; !ok { - if checkBaseComponentExists { - errorMessage := fmt.Sprintf("The component '%[1]s' in the stack manifest '%[2]s' inherits from '%[3]s' "+ - "(using 'metadata.inherits'), but '%[3]s' is not defined in any of the config files for the stack '%[2]s'", - component, - stack, - baseComponentFromInheritList, - ) - return errors.New(errorMessage) - } - } - - // Process the baseComponentFromInheritList components recursively to find `componentInheritanceChain` - err := ProcessBaseComponentConfig( - cliConfig, - baseComponentConfig, - allComponentsMap, - component, - stack, - baseComponentFromInheritList, - componentBasePath, - checkBaseComponentExists, - baseComponents, - ) - if err != nil { - return err - } - } - } - } - - if baseComponentVarsSection, baseComponentVarsSectionExist := baseComponentMap["vars"]; baseComponentVarsSectionExist { - baseComponentVars, ok = baseComponentVarsSection.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.vars' section in the stack '%s'", baseComponent, stack) - } - } - - if baseComponentSettingsSection, baseComponentSettingsSectionExist := baseComponentMap["settings"]; baseComponentSettingsSectionExist { - baseComponentSettings, ok = baseComponentSettingsSection.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.settings' section in the stack '%s'", baseComponent, stack) - } - } - - if baseComponentEnvSection, baseComponentEnvSectionExist := baseComponentMap["env"]; baseComponentEnvSectionExist { - baseComponentEnv, ok = baseComponentEnvSection.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.env' section in the stack '%s'", baseComponent, stack) - } - } - - if baseComponentProvidersSection, baseComponentProvidersSectionExist := baseComponentMap[cfg.ProvidersSectionName]; baseComponentProvidersSectionExist { - baseComponentProviders, ok = baseComponentProvidersSection.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.providers' section in the stack '%s'", baseComponent, stack) - } - } - - // Base component backend - if i, ok2 := baseComponentMap["backend_type"]; ok2 { - baseComponentBackendType, ok = i.(string) - if !ok { - return fmt.Errorf("invalid '%s.backend_type' section in the stack '%s'", baseComponent, stack) - } - } - - if i, ok2 := baseComponentMap["backend"]; ok2 { - baseComponentBackendSection, ok = i.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.backend' section in the stack '%s'", baseComponent, stack) - } - } - - // Base component remote state backend - if i, ok2 := baseComponentMap["remote_state_backend_type"]; ok2 { - baseComponentRemoteStateBackendType, ok = i.(string) - if !ok { - return fmt.Errorf("invalid '%s.remote_state_backend_type' section in the stack '%s'", baseComponent, stack) - } - } - - if i, ok2 := baseComponentMap["remote_state_backend"]; ok2 { - baseComponentRemoteStateBackendSection, ok = i.(map[any]any) - if !ok { - return fmt.Errorf("invalid '%s.remote_state_backend' section in the stack '%s'", baseComponent, stack) - } - } - - // Base component `command` - if baseComponentCommandSection, baseComponentCommandSectionExist := baseComponentMap[cfg.CommandSectionName]; baseComponentCommandSectionExist { - baseComponentCommand, ok = baseComponentCommandSection.(string) - if !ok { - return fmt.Errorf("invalid '%s.command' section in the stack '%s'", baseComponent, stack) - } - } - - if len(baseComponentConfig.FinalBaseComponentName) == 0 { - baseComponentConfig.FinalBaseComponentName = baseComponent - } - - // Base component `vars` - merged, err := m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentVars, baseComponentVars}) - if err != nil { - return err - } - baseComponentConfig.BaseComponentVars = merged - - // Base component `settings` - merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentSettings, baseComponentSettings}) - if err != nil { - return err - } - baseComponentConfig.BaseComponentSettings = merged - - // Base component `env` - merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentEnv, baseComponentEnv}) - if err != nil { - return err - } - baseComponentConfig.BaseComponentEnv = merged - - // Base component `providers` - merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentProviders, baseComponentProviders}) - if err != nil { - return err - } - baseComponentConfig.BaseComponentProviders = merged - - // Base component `command` - baseComponentConfig.BaseComponentCommand = baseComponentCommand - - // Base component `backend_type` - baseComponentConfig.BaseComponentBackendType = baseComponentBackendType - - // Base component `backend` - merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentBackendSection, baseComponentBackendSection}) - if err != nil { - return err - } - baseComponentConfig.BaseComponentBackendSection = merged - - // Base component `remote_state_backend_type` - baseComponentConfig.BaseComponentRemoteStateBackendType = baseComponentRemoteStateBackendType - - // Base component `remote_state_backend` - merged, err = m.Merge(cliConfig, []map[any]any{baseComponentConfig.BaseComponentRemoteStateBackendSection, baseComponentRemoteStateBackendSection}) - if err != nil { - return err - } - baseComponentConfig.BaseComponentRemoteStateBackendSection = merged - - baseComponentConfig.ComponentInheritanceChain = u.UniqueStrings(append([]string{baseComponent}, baseComponentConfig.ComponentInheritanceChain...)) - } else { - if checkBaseComponentExists { - // Check if the base component exists as Terraform/Helmfile component - // If it does exist, don't throw errors if it is not defined in YAML config - componentPath := path.Join(componentBasePath, baseComponent) - componentPathExists, err := u.IsDirectory(componentPath) - if err != nil || !componentPathExists { - return errors.New("The component '" + component + "' inherits from the base component '" + - baseComponent + "' (using 'component:' attribute), " + "but `" + baseComponent + "' is not defined in any of the YAML config files for the stack '" + stack + "'") - } - } - } - - return nil -} - -// FindComponentsDerivedFromBaseComponents finds all components that derive from the given base components -func FindComponentsDerivedFromBaseComponents( - stack string, - allComponents map[string]any, - baseComponents []string, -) ([]string, error) { - - res := []string{} - - for component, compSection := range allComponents { - componentSection, ok := compSection.(map[string]any) - if !ok { - return nil, fmt.Errorf("invalid '%s' component section in the file '%s'", component, stack) - } - - if base, baseComponentExist := componentSection[cfg.ComponentSectionName]; baseComponentExist { - baseComponent, ok := base.(string) - if !ok { - return nil, fmt.Errorf("invalid 'component' attribute in the component '%s' in the file '%s'", component, stack) - } - - if baseComponent != "" && u.SliceContainsString(baseComponents, baseComponent) { - res = append(res, component) - } - } - } - - return res, nil -} diff --git a/pkg/utils/json_utils.go b/pkg/utils/json_utils.go index 92bc7ec4e..ffcb14df0 100644 --- a/pkg/utils/json_utils.go +++ b/pkg/utils/json_utils.go @@ -1,11 +1,12 @@ package utils import ( - "github.com/cloudposse/atmos/pkg/schema" "os" "strings" jsoniter "github.com/json-iterator/go" + + "github.com/cloudposse/atmos/pkg/schema" ) // PrintAsJSON prints the provided value as JSON document to the console diff --git a/website/docs/cli/commands/describe/describe-component.mdx b/website/docs/cli/commands/describe/describe-component.mdx index 0f957af2a..01cb3fa68 100644 --- a/website/docs/cli/commands/describe/describe-component.mdx +++ b/website/docs/cli/commands/describe/describe-component.mdx @@ -63,7 +63,7 @@ atmos describe component test/test-component-override -s tenant2-ue2-prod ## Output -The command outputs the final deep-merged component configuration in YAML format. +The command outputs the final deep-merged component configuration. The output contains the following sections: @@ -75,19 +75,23 @@ The output contains the following sections: - `atmos_stack` - [Atmos stack](/core-concepts/stacks) name +- `stack` - same as `atmos_stack` + - `atmos_stack_file` - the stack manifest where the Atmos stack is defined -- `backend` - Terraform backend configuration +- `atmos_manifest` - same as `atmos_stack_file` + +- `backend` - Terraform/OpenTofu backend configuration -- `backend_type` - Terraform backend type +- `backend_type` - Terraform/OpenTofu backend type -- `command` - the binary to execute when provisioning the component (e.g. `terraform`, `terraform-1`, `helmfile`) +- `command` - the binary to execute when provisioning the component (e.g. `terraform`, `terraform-1`, `tofu`, `helmfile`) -- `component` - the Terraform component for which the Atmos component provides configuration +- `component` - the Terraform/OpenTofu component for which the Atmos component provides configuration - `component_info` - a block describing the Terraform or Helmfile components that the Atmos component manages. The `component_info` block has the following sections: - - `component_path` - the filesystem path to the Terraform or Helmfile component + - `component_path` - the filesystem path to the Terraform/OpenTofu or Helmfile component - `component_type` - the type of the component (`terraform` or `helmfile`) @@ -102,9 +106,9 @@ The output contains the following sections: - `metadata` - component's metadata config -- `remote_state_backend` - Terraform backend config for remote state +- `remote_state_backend` - Terraform/OpenTofu backend config for remote state -- `remote_state_backend_type` - Terraform backend type for remote state +- `remote_state_backend_type` - Terraform/OpenTofu backend type for remote state - `settings` - component settings (free-form map) @@ -113,10 +117,10 @@ The output contains the following sections: - `spacelift_stack` - Spacelift stack name (if [Spacelift Integration](/integrations/spacelift) is configured for the component in the stack and `settings.spacelift.workspace_enabled` is set to `true`) -- `vars` - the final deep-merged component variables that are provided to Terraform and Helmfile when executing `atmos terraform` - and `atmos helmfile` commands +- `vars` - the final deep-merged component variables that are provided to Terraform/OpenTofu and Helmfile when executing + `atmos terraform` and `atmos helmfile` commands -- `workspace` - Terraform workspace for the Atmos component +- `workspace` - Terraform/OpenTofu workspace for the Atmos component - `imports` - a list of all imports in the Atmos stack (this shows all imports in the stack, related to the component and not) @@ -161,7 +165,7 @@ The difference between the `imports`, `deps_all` and `deps` outputs is as follow terraform: derived-component-3: metadata: - component: "test/test-component" # Point to the Terraform component + component: "test/test-component" # Point to the Terraform/OpenTofu component inherits: # Inherit all the values from the base component - base-component-4 diff --git a/website/docs/core-concepts/components/remote-state.mdx b/website/docs/core-concepts/components/remote-state.mdx index b8e57fb34..c2ded31a3 100644 --- a/website/docs/core-concepts/components/remote-state.mdx +++ b/website/docs/core-concepts/components/remote-state.mdx @@ -337,3 +337,11 @@ For this to work for both the `atmos` CLI and the Terraform provider, we recomme the correct stack. For example, if the component is provisioned in a different AWS region (let's say `us-west-2`), we can set `environment = "uw2"`, and the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) module will get the remote state outputs for the Atmos component provisioned in that region + +## Read remote state by using `atmos.Component` `Go` template function + +Atmos supports an alternative way to read the outputs (remote state) of components directly in Atmos stack manifests by +using the `atmos.Component` `Go` template function and the `outputs` section instead of using the `remote-state` +module and configuring Terraform/OpenTofu components to use the module. + +For more details, refer to [`atmos.Component` function](/core-concepts/template-functions/atmos.Component). diff --git a/website/docs/core-concepts/custom-commands/custom-commands.mdx b/website/docs/core-concepts/custom-commands/custom-commands.mdx index e2a57158b..f6e337d27 100644 --- a/website/docs/core-concepts/custom-commands/custom-commands.mdx +++ b/website/docs/core-concepts/custom-commands/custom-commands.mdx @@ -1,8 +1,10 @@ --- title: Atmos Custom Commands sidebar_label: Custom Commands +sidebar_position: 5 id: custom-commands --- + import File from '@site/src/components/File' Atmos can be easily extended to support any number of custom CLI commands. diff --git a/website/docs/core-concepts/stacks/stacks.mdx b/website/docs/core-concepts/stacks/stacks.mdx index b4eabd614..7e332397e 100644 --- a/website/docs/core-concepts/stacks/stacks.mdx +++ b/website/docs/core-concepts/stacks/stacks.mdx @@ -1,10 +1,11 @@ --- title: Atmos Stacks -sidebar_position: 1 +sidebar_position: 2 sidebar_label: Stacks description: Stacks are a way to express the complete infrastructure needed for an environment id: stacks --- + import File from '@site/src/components/File' Stacks are a way to express the complete infrastructure needed for an environment. Think of a Stack like an architectural "Blueprint" composed diff --git a/website/docs/core-concepts/stacks/templating.mdx b/website/docs/core-concepts/stacks/templating.mdx index b2f4e3019..e30b77b55 100644 --- a/website/docs/core-concepts/stacks/templating.mdx +++ b/website/docs/core-concepts/stacks/templating.mdx @@ -9,9 +9,13 @@ import Terminal from '@site/src/components/Terminal' Atmos supports [Go templates](https://pkg.go.dev/text/template) in stack manifests. -[Sprig Functions](https://masterminds.github.io/sprig/), [Gomplate Functions](https://docs.gomplate.ca/functions/) -and [Gomplate Datasources](https://docs.gomplate.ca/datasources/) -are supported as well. +In `Go` templates, you can use the following functions and datasources: + + - [Go `text/template` functions](https://pkg.go.dev/text/template#hdr-Functions) + - [Sprig Functions](https://masterminds.github.io/sprig/) + - [Gomplate Functions](https://docs.gomplate.ca/functions/) + - [Gomplate Datasources](https://docs.gomplate.ca/datasources/) + - [Atmos Template Functions](/core-concepts/template-functions) ## Important Note diff --git a/website/docs/core-concepts/template-functions/atmos.Component.mdx b/website/docs/core-concepts/template-functions/atmos.Component.mdx new file mode 100644 index 000000000..73cc881c8 --- /dev/null +++ b/website/docs/core-concepts/template-functions/atmos.Component.mdx @@ -0,0 +1,157 @@ +--- +title: atmos.Component +sidebar_position: 1 +sidebar_label: atmos.Component +--- + +import File from '@site/src/components/File' + +The `atmos.Component` template function allows reading any Atmos section or any attribute from a section for an +Atmos component in a stack, and use it in `Go` templates in Atmos component configurations. + +## Usage + +```yaml + {{ (atmos.Component "" "").
. }} +``` + +## Arguments + +- `component` - Atmos component name + +- `stack` - Atmos stack name + +- `section` - Atmos section name. Any section returned by the CLI command + [`atmos describe component`](/cli/commands/describe/component#output) can be used. + A special `outputs` section is also supported to get the outputs (remote state) of Terraform/OpenTofu components. + + :::note + Using the `outputs` section in the `atmos.Component` command is an alternative way to read the outputs (remote state) + of a component in a stack directly in Atmos stack manifests instead of using the `remote-state` + module and configuring Terraform/OpenTofu components to use the `remote-state` module as described in + [Component Remote State](/core-concepts/components/remote-state) + ::: + +
+ +- `attribute` - attribute name (field) from the `section`. `attribute` is optional, you can use the `section` itself + if it's a simple type (e.g. `string`). Any number of attributes can be chained using the dot (`.`) notation. + For example, if the first two attributes are maps, you can chain them and get a field from the last map: + + ```yaml + {{ (atmos.Component "" "").
... }} + ``` + +## Specifying Atmos `stack` + +`stack` is the second argument of the `atmos.Component` function, and it can be specified in a few different ways: + + - Hardcoded stack name. Use it if you want to get an output from a component from a different (well-known and static) + stack. For example, you have a `tgw` component in a stack `plat-ue2-dev` that requires the `vpc_id` output from + the `vpc` component from the stack `plat-ue2-prod`: + + ```yaml title="plat-ue2-dev" + components: + terraform: + tgw: + vars: + vpc_id: '{{ (atmos.Component "vpc" "plat-ue2-prod").outputs.vpc_id }}' + ``` + + - Use the `.stack` (or `.atmos_stack`) template identifier to specify the same stack as the current component + (for which the `atmos.Component` function is executed): + + ```yaml + {{ (atmos.Component "" .stack).
. }} + {{ (atmos.Component "" .atmos_stack).
. }} + ``` + + For example, you have a `tgw` component that requires the `vpc_id` output from the `vpc` component in the same stack: + + ```yaml + components: + terraform: + tgw: + vars: + vpc_id: '{{ (atmos.Component "vpc" .stack).outputs.vpc_id }}' + ``` + + - Use the `printf` template function to construct stack names using static strings and dynamic identifiers: + + ```yaml + {{ (atmos.Component "" (printf "%s-%s-%s" .vars.tenant .vars.environment .vars.stage)).
. }} + + {{ (atmos.Component "" (printf "plat-%s-prod" .vars.environment)).
. }} + + {{ (atmos.Component "" (printf "%s-%s-%s" .settings.context.tenant .settings.context.region .settings.context.account)).
. }} + ``` + + For example, you have a `tgw` component deployed in the stack `plat-ue2-dev`. The `tgw` component requires the + `vpc_id` output from the `vpc` component from the same environment (`ue2`) and same stage (`dev`), but from a different + tenant `net` (instead of `plat`): + + ```yaml title="plat-ue2-dev" + components: + terraform: + tgw: + vars: + vpc_id: '{{ (atmos.Component "vpc" (printf "net-%s-%s" .vars.environment .vars.stage)).outputs.vpc_id }}' + ``` + +
+ + :::tip Important + By using the `printf "%s-%s-%s"` function, you are constructing stack names using the stack context variables/identifiers. + + For more information on Atmos stack names and how to define them, refer to `stacks.name_pattern` and `stacks.name_template` + sections in [`atmos.yaml` CLI config file](/cli/configuration/#stacks) + ::: + +## Examples + +The following configurations show different ways of using the `atmos.Component` template function to read values from +different Atmos sections directly in Atmos stack manifests, including the outputs of other +(already provisioned) components. + + +```yaml +# Global `settings` section +# It will be added and deep-merged to the `settings` section of all components +settings: + test: true + +components: + terraform: + test: + metadata: + # Point to the Terraform/OpenTofu component + component: "test" + vars: + name: "test" + + test1: + metadata: + # Point to the Terraform/OpenTofu component + component: "test1" + vars: + name: "test1" + + test2: + metadata: + # Point to the Terraform/OpenTofu component + component: "test2" + vars: + name: "test2" + # Use the `atmos.Component` function to get the outputs of the Atmos component `test1` + # The `test1` component must be already provisioned and its outputs stored in the Terraform/OpenTofu state + # Atmos will execute `terraform output` on the `test1` component in the same stack to read its outputs + test1_id: '{{ (atmos.Component "test1" .stack).outputs.test1_id }}' + tags: + # Get the `settings.test` field from the `test` component in the same stack + test: '{{ (atmos.Component "test" .stack).settings.test }}' + # Get the `metadata.component` field from the `test` component in the same stack + test_terraform_component: '{{ (atmos.Component "test" .stack).metadata.component }}' + # Get the `vars.name` field from the `test1` component in the same stack + test1_name: '{{ (atmos.Component "test1" .stack).vars.name }}' +``` + diff --git a/website/docs/core-concepts/template-functions/template-functions.mdx b/website/docs/core-concepts/template-functions/template-functions.mdx new file mode 100644 index 000000000..af68bf2f3 --- /dev/null +++ b/website/docs/core-concepts/template-functions/template-functions.mdx @@ -0,0 +1,17 @@ +--- +title: Atmos Template Functions +sidebar_position: 6 +sidebar_label: Template Functions +description: Atmos functions for the Go’s template language. +id: template-functions +--- + +Atmos provides template functions for the `Go` template language. + +These functions can be used in [`Go` templates in Atmos stack manifests](/core-concepts/stacks/templating). + +
+ +import DocCardList from "@theme/DocCardList"; + + diff --git a/website/docs/core-concepts/vendoring/vendoring.mdx b/website/docs/core-concepts/vendoring/vendoring.mdx index be94ad088..5bbc50107 100644 --- a/website/docs/core-concepts/vendoring/vendoring.mdx +++ b/website/docs/core-concepts/vendoring/vendoring.mdx @@ -1,7 +1,7 @@ --- title: Vendoring description: Use Atmos vendoring to make copies of 3rd-party components, stacks, and other artifacts in your own repo. -sidebar_position: 14 +sidebar_position: 4 sidebar_label: Vendoring id: vendoring --- diff --git a/website/docs/core-concepts/workflows/workflows.mdx b/website/docs/core-concepts/workflows/workflows.mdx index b394102be..6123dbbf8 100644 --- a/website/docs/core-concepts/workflows/workflows.mdx +++ b/website/docs/core-concepts/workflows/workflows.mdx @@ -1,8 +1,9 @@ --- title: Workflows -sidebar_position: 12 +sidebar_position: 3 sidebar_label: Workflows --- + import File from '@site/src/components/File' import Terminal from '@site/src/components/Terminal' diff --git a/website/docs/integrations/atlantis.mdx b/website/docs/integrations/atlantis.mdx index 30882836c..862eb9de5 100644 --- a/website/docs/integrations/atlantis.mdx +++ b/website/docs/integrations/atlantis.mdx @@ -687,7 +687,7 @@ on: branches: [ main ] env: - ATMOS_VERSION: 1.80.0 + ATMOS_VERSION: 1.81.0 ATMOS_CLI_CONFIG_PATH: ./ jobs: diff --git a/website/docs/integrations/github-actions/setup-atmos.mdx b/website/docs/integrations/github-actions/setup-atmos.mdx index ff2240d5a..51de036d3 100644 --- a/website/docs/integrations/github-actions/setup-atmos.mdx +++ b/website/docs/integrations/github-actions/setup-atmos.mdx @@ -30,6 +30,6 @@ jobs: uses: cloudposse/github-action-setup-atmos with: # Make sure to pin to the latest version of atmos - atmos_version: 1.80.0 + atmos_version: 1.81.0 ``` diff --git a/website/package-lock.json b/website/package-lock.json index db25fe0c2..cffc70f4b 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -25,7 +25,7 @@ "custom-loaders": "file:plugins/custom-loaders", "docusaurus-plugin-image-zoom": "^2.0.0", "html-loader": "^5.0.0", - "marked": "^12.0.2", + "marked": "^13.0.0", "prism-react-renderer": "^2.3.1", "react": "^18.3.1", "react-dom": "^18.3.1", @@ -9330,9 +9330,10 @@ } }, "node_modules/marked": { - "version": "12.0.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-12.0.2.tgz", - "integrity": "sha512-qXUm7e/YKFoqFPYPa3Ukg9xlI5cyAtGmyEIzMfW//m6kXwCy2Ps9DYf5ioijFKQ8qyuscrHoY04iJGctu2Kg0Q==", + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-13.0.0.tgz", + "integrity": "sha512-VTeDCd9txf4KLLljUZ0nljE/Incb9SrWuueE44QVuU0pkOdh4sfCeW1Z6lPcxyDRSVY6rm8db/0OPaN75RNUmw==", + "license": "MIT", "bin": { "marked": "bin/marked.js" }, diff --git a/website/package.json b/website/package.json index 6d9b1c6f2..7b5a412ee 100644 --- a/website/package.json +++ b/website/package.json @@ -31,7 +31,7 @@ "custom-loaders": "file:plugins/custom-loaders", "docusaurus-plugin-image-zoom": "^2.0.0", "html-loader": "^5.0.0", - "marked": "^12.0.2", + "marked": "^13.0.0", "prism-react-renderer": "^2.3.1", "react": "^18.3.1", "react-dom": "^18.3.1",