Improve atmos validate stacks and atmos describe affected commands (

#608)

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

* updates

cmd/describe_affected.go

@@ -30,7 +30,7 @@ func init() {
 	describeAffectedCmd.PersistentFlags().String("repo-path", "", "Filesystem path to the already cloned target repository with which to compare the current branch: atmos describe affected --repo-path <path_to_already_cloned_repo>")
 	describeAffectedCmd.PersistentFlags().String("ref", "", "Git reference with which to compare the current branch: atmos describe affected --ref refs/heads/main. Refer to https://git-scm.com/book/en/v2/Git-Internals-Git-References for more details")
 	describeAffectedCmd.PersistentFlags().String("sha", "", "Git commit SHA with which to compare the current branch: atmos describe affected --sha 3a5eafeab90426bd82bf5899896b28cc0bab3073")
-	describeAffectedCmd.PersistentFlags().String("file", "", "Write the result to the file: atmos describe affected --ref refs/tags/v1.71.0 --file affected.json")
+	describeAffectedCmd.PersistentFlags().String("file", "", "Write the result to the file: atmos describe affected --ref refs/tags/v1.75.0 --file affected.json")
 	describeAffectedCmd.PersistentFlags().String("format", "json", "The output format: atmos describe affected --format=json|yaml ('json' is default)")
 	describeAffectedCmd.PersistentFlags().Bool("verbose", false, "Print more detailed output when cloning and checking out the Git repository: atmos describe affected --verbose=true")
 	describeAffectedCmd.PersistentFlags().String("ssh-key", "", "Path to PEM-encoded private key to clone private repos using SSH: atmos describe affected --ssh-key <path_to_ssh_key>")

examples/quick-start/Dockerfile

@@ -1,15 +1,15 @@
 # Geodesic: https://github.com/cloudposse/geodesic/
-ARG GEODESIC_VERSION=2.11.2
+ARG GEODESIC_VERSION=2.11.3
 ARG GEODESIC_OS=debian
 
 # Atmos
 # https://atmos.tools/
 # https://github.com/cloudposse/atmos
 # https://github.com/cloudposse/atmos/releases
-ARG ATMOS_VERSION=1.74.0
+ARG ATMOS_VERSION=1.75.0
 
 # Terraform: https://github.com/hashicorp/terraform/releases
-ARG TF_VERSION=1.8.1
+ARG TF_VERSION=1.8.4
 
 FROM cloudposse/geodesic:${GEODESIC_VERSION}-${GEODESIC_OS}
 

go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/arsham/figurine v1.3.0
 	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/charmbracelet/bubbles v0.18.0
-	github.com/charmbracelet/bubbletea v0.26.2
-	github.com/charmbracelet/lipgloss v0.10.0
+	github.com/charmbracelet/bubbletea v0.26.3
+	github.com/charmbracelet/lipgloss v0.11.0
 	github.com/elewis787/boa v0.1.2
 	github.com/fatih/color v1.17.0
 	github.com/go-git/go-git/v5 v5.12.0
@@ -90,6 +90,10 @@ require (
 	github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/charmbracelet/x/ansi v0.1.1 // indirect
+	github.com/charmbracelet/x/input v0.1.0 // indirect
+	github.com/charmbracelet/x/term v0.1.1 // indirect
+	github.com/charmbracelet/x/windows v0.1.0 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
 	github.com/containerd/containerd v1.7.15 // indirect
@@ -207,6 +211,7 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/yashtewari/glob-intersection v0.2.0 // indirect
 	github.com/zealic/xignore v0.3.3 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect

go.sum

@@ -386,10 +386,18 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0=
 github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw=
-github.com/charmbracelet/bubbletea v0.26.2 h1:Eeb+n75Om9gQ+I6YpbCXQRKHt5Pn4vMwusQpwLiEgJQ=
-github.com/charmbracelet/bubbletea v0.26.2/go.mod h1:6I0nZ3YHUrQj7YHIHlM8RySX4ZIthTliMY+W8X8b+Gs=
-github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s=
-github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE=
+github.com/charmbracelet/bubbletea v0.26.3 h1:iXyGvI+FfOWqkB2V07m1DF3xxQijxjY2j8PqiXYqasg=
+github.com/charmbracelet/bubbletea v0.26.3/go.mod h1:bpZHfDHTYJC5g+FBK+ptJRCQotRC+Dhh3AoMxa/2+3Q=
+github.com/charmbracelet/lipgloss v0.11.0 h1:UoAcbQ6Qml8hDwSWs0Y1cB5TEQuZkDPH/ZqwWWYTG4g=
+github.com/charmbracelet/lipgloss v0.11.0/go.mod h1:1UdRTH9gYgpcdNN5oBtjbu/IzNKtzVtb7sqN1t9LNn8=
+github.com/charmbracelet/x/ansi v0.1.1 h1:CGAduulr6egay/YVbGc8Hsu8deMg1xZ/bkaXTPi1JDk=
+github.com/charmbracelet/x/ansi v0.1.1/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
+github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ=
+github.com/charmbracelet/x/input v0.1.0/go.mod h1:ZZwaBxPF7IG8gWWzPUVqHEtWhc1+HXJPNuerJGRGZ28=
+github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI=
+github.com/charmbracelet/x/term v0.1.1/go.mod h1:wB1fHt5ECsu3mXYusyzcngVWWlu1KKUmmLhfgr/Flxw=
+github.com/charmbracelet/x/windows v0.1.0 h1:gTaxdvzDM5oMa/I2ZNF7wN78X/atWemG9Wph7Ika2k4=
+github.com/charmbracelet/x/windows v0.1.0/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -1160,6 +1168,8 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMc
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg=
 github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/exec/describe_affected.go

@@ -23,6 +23,11 @@ func ExecuteDescribeAffectedCmd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	err = ValidateStacks(cliConfig)
+	if err != nil {
+		return err
+	}
+
 	// Process flags
 	flags := cmd.Flags()
 
@@ -102,6 +107,10 @@ func ExecuteDescribeAffectedCmd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	if verbose {
+		cliConfig.Logs.Level = u.LogLevelTrace
+	}
+
 	u.LogTrace(cliConfig, fmt.Sprintf("\nAffected components and stacks: \n"))
 
 	err = printOrWriteToFile(format, file, affected)

internal/exec/describe_affected_utils.go

@@ -492,21 +492,27 @@ func executeDescribeAffected(
 	}
 
 	u.LogTrace(cliConfig, fmt.Sprintf("Got remote repo commit tree"))
-	u.LogTrace(cliConfig, fmt.Sprintf("Finding diff between the current working branch and remote target branch ..."))
+	u.LogTrace(cliConfig, fmt.Sprintf("Finding difference between the current working branch and remote target branch ..."))
 
 	// Find a slice of Patch objects with all the changes between the current working and remote trees
 	patch, err := localTree.Patch(remoteTree)
 	if err != nil {
 		return nil, err
 	}
 
-	u.LogTrace(cliConfig, fmt.Sprintf("Found diff between the current working branch and remote target branch"))
-	u.LogTrace(cliConfig, "\nChanged files:\n")
-
 	var changedFiles []string
-	for _, fileStat := range patch.Stats() {
-		u.LogTrace(cliConfig, fileStat.Name)
-		changedFiles = append(changedFiles, fileStat.Name)
+
+	if len(patch.Stats()) > 0 {
+		u.LogTrace(cliConfig, fmt.Sprintf("Found difference between the current working branch and remote target branch"))
+		u.LogTrace(cliConfig, "\nChanged files:\n")
+
+		for _, fileStat := range patch.Stats() {
+			u.LogTrace(cliConfig, fileStat.Name)
+			changedFiles = append(changedFiles, fileStat.Name)
+		}
+		u.LogTrace(cliConfig, "")
+	} else {
+		u.LogTrace(cliConfig, fmt.Sprintf("The current working branch and remote target branch are the same"))
 	}
 
 	affected, err := findAffected(currentStacks, remoteStacks, cliConfig, changedFiles, includeSpaceliftAdminStacks)

internal/exec/describe_dependents.go

@@ -25,6 +25,11 @@ func ExecuteDescribeDependentsCmd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	err = ValidateStacks(cliConfig)
+	if err != nil {
+		return err
+	}
+
 	if len(args) != 1 {
 		return errors.New("invalid arguments. The command requires one argument `component`")
 	}

internal/exec/describe_stacks.go

@@ -27,6 +27,11 @@ func ExecuteDescribeStacksCmd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	err = ValidateStacks(cliConfig)
+	if err != nil {
+		return err
+	}
+
 	flags := cmd.Flags()
 
 	filterByStack, err := flags.GetString("stack")

internal/exec/validate_stacks.go

@@ -9,6 +9,7 @@ import (
 	"github.com/spf13/cobra"
 
 	cfg "github.com/cloudposse/atmos/pkg/config"
+	"github.com/cloudposse/atmos/pkg/schema"
 	s "github.com/cloudposse/atmos/pkg/stack"
 	u "github.com/cloudposse/atmos/pkg/utils"
 )
@@ -36,6 +37,44 @@ func ExecuteValidateStacksCmd(cmd *cobra.Command, args []string) error {
 		cliConfig.Schemas.Atmos.Manifest = schemasAtmosManifestFlag
 	}
 
+	return ValidateStacks(cliConfig)
+}
+
+// ValidateStacks validates Atmos stack configuration
+func ValidateStacks(cliConfig schema.CliConfiguration) error {
+	var validationErrorMessages []string
+
+	// 1. Process top-level stack manifests and detect duplicate components in the same stack
+	stacksMap, _, err := FindStacksMap(cliConfig, false)
+	if err != nil {
+		return err
+	}
+
+	terraformComponentStackMap, err := createComponentStackMap(cliConfig, stacksMap, cfg.TerraformSectionName)
+	if err != nil {
+		return err
+	}
+
+	errorList, err := checkComponentStackMap(terraformComponentStackMap)
+	if err != nil {
+		return err
+	}
+	validationErrorMessages = append(validationErrorMessages, errorList...)
+
+	helmfileComponentStackMap, err := createComponentStackMap(cliConfig, stacksMap, cfg.HelmfileSectionName)
+	if err != nil {
+		return err
+	}
+
+	errorList, err = checkComponentStackMap(helmfileComponentStackMap)
+	if err != nil {
+		return err
+	}
+	validationErrorMessages = append(validationErrorMessages, errorList...)
+
+	// 2. Check all YAML stack manifests defined in the infrastructure
+	// It will check YAML syntax and all the Atmos sections defined in the manifests
+
 	// Check if the Atmos manifest JSON Schema is configured and the file exists
 	// The path to the Atmos manifest JSON Schema can be absolute path or a path relative to the `base_path` setting in `atmos.yaml`
 	var atmosManifestJsonSchemaFilePath string
@@ -73,8 +112,6 @@ func ExecuteValidateStacksCmd(cmd *cobra.Command, args []string) error {
 	u.LogDebug(cliConfig, fmt.Sprintf("Validating all YAML files in the '%s' folder and all subfolders\n",
 		path.Join(cliConfig.BasePath, cliConfig.Stacks.BasePath)))
 
-	var errorMessages []string
-
 	for _, filePath := range stackConfigFilesAbsolutePaths {
 		stackConfig, importsConfig, _, err := s.ProcessYAMLConfigFile(
 			cliConfig,
@@ -91,11 +128,10 @@ func ExecuteValidateStacksCmd(cmd *cobra.Command, args []string) error {
 			atmosManifestJsonSchemaFilePath,
 		)
 		if err != nil {
-			errorMessages = append(errorMessages, err.Error())
+			validationErrorMessages = append(validationErrorMessages, err.Error())
 		}
 
 		// Process and validate the stack manifest
-		componentStackMap := map[string]map[string][]string{}
 		_, err = s.ProcessStackConfig(
 			cliConfig,
 			cliConfig.StacksBaseAbsolutePath,
@@ -106,17 +142,146 @@ func ExecuteValidateStacksCmd(cmd *cobra.Command, args []string) error {
 			false,
 			true,
 			"",
-			componentStackMap,
+			map[string]map[string][]string{},
 			importsConfig,
-			false)
+			false,
+		)
 		if err != nil {
-			errorMessages = append(errorMessages, err.Error())
+			validationErrorMessages = append(validationErrorMessages, err.Error())
 		}
 	}
 
-	if len(errorMessages) > 0 {
-		return errors.New(strings.Join(errorMessages, "\n\n"))
+	if len(validationErrorMessages) > 0 {
+		return errors.New(strings.Join(validationErrorMessages, "\n\n"))
 	}
 
 	return nil
 }
+
+func createComponentStackMap(
+	cliConfig schema.CliConfiguration,
+	stacksMap map[string]any,
+	componentType string,
+) (map[string]map[string][]string, error) {
+	var varsSection map[any]any
+	var metadataSection map[any]any
+	var settingsSection map[any]any
+	var envSection map[any]any
+	var providersSection map[any]any
+	var overridesSection map[any]any
+	var backendSection map[any]any
+	var backendTypeSection string
+	var stackName string
+	var err error
+	terraformComponentStackMap := make(map[string]map[string][]string)
+
+	for stackManifest, stackSection := range stacksMap {
+		if componentsSection, ok := stackSection.(map[any]any)[cfg.ComponentsSectionName].(map[string]any); ok {
+
+			// Terraform components
+			if terraformSection, ok := componentsSection[componentType].(map[string]any); ok {
+				for componentName, compSection := range terraformSection {
+					componentSection, ok := compSection.(map[string]any)
+
+					if varsSection, ok = componentSection[cfg.VarsSectionName].(map[any]any); !ok {
+						varsSection = map[any]any{}
+					}
+
+					if metadataSection, ok = componentSection[cfg.MetadataSectionName].(map[any]any); !ok {
+						metadataSection = map[any]any{}
+					}
+
+					if settingsSection, ok = componentSection[cfg.SettingsSectionName].(map[any]any); !ok {
+						settingsSection = map[any]any{}
+					}
+
+					if envSection, ok = componentSection[cfg.EnvSectionName].(map[any]any); !ok {
+						envSection = map[any]any{}
+					}
+
+					if providersSection, ok = componentSection[cfg.ProvidersSectionName].(map[any]any); !ok {
+						providersSection = map[any]any{}
+					}
+
+					if overridesSection, ok = componentSection[cfg.OverridesSectionName].(map[any]any); !ok {
+						overridesSection = map[any]any{}
+					}
+
+					if backendSection, ok = componentSection[cfg.BackendSectionName].(map[any]any); !ok {
+						backendSection = map[any]any{}
+					}
+
+					if backendTypeSection, ok = componentSection[cfg.BackendTypeSectionName].(string); !ok {
+						backendTypeSection = ""
+					}
+
+					configAndStacksInfo := schema.ConfigAndStacksInfo{
+						ComponentFromArg:          componentName,
+						Stack:                     stackName,
+						ComponentMetadataSection:  metadataSection,
+						ComponentVarsSection:      varsSection,
+						ComponentSettingsSection:  settingsSection,
+						ComponentEnvSection:       envSection,
+						ComponentProvidersSection: providersSection,
+						ComponentOverridesSection: overridesSection,
+						ComponentBackendSection:   backendSection,
+						ComponentBackendType:      backendTypeSection,
+						ComponentSection: map[string]any{
+							cfg.VarsSectionName:        varsSection,
+							cfg.MetadataSectionName:    metadataSection,
+							cfg.SettingsSectionName:    settingsSection,
+							cfg.EnvSectionName:         envSection,
+							cfg.ProvidersSectionName:   providersSection,
+							cfg.OverridesSectionName:   overridesSection,
+							cfg.BackendSectionName:     backendSection,
+							cfg.BackendTypeSectionName: backendTypeSection,
+						},
+					}
+
+					// Find Atmos stack name
+					if cliConfig.Stacks.NameTemplate != "" {
+						stackName, err = u.ProcessTmpl("validate-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false)
+						if err != nil {
+							return nil, err
+						}
+					} else {
+						context := cfg.GetContextFromVars(varsSection)
+						configAndStacksInfo.Context = context
+						stackName, err = cfg.GetContextPrefix(stackManifest, context, GetStackNamePattern(cliConfig), stackManifest)
+						if err != nil {
+							return nil, err
+						}
+					}
+
+					_, ok = terraformComponentStackMap[componentName]
+					if !ok {
+						terraformComponentStackMap[componentName] = make(map[string][]string)
+					}
+					terraformComponentStackMap[componentName][stackName] = append(terraformComponentStackMap[componentName][stackName], stackManifest)
+				}
+			}
+		}
+	}
+
+	return terraformComponentStackMap, nil
+}
+
+func checkComponentStackMap(componentStackMap map[string]map[string][]string) ([]string, error) {
+	var res []string
+
+	for componentName, componentSection := range componentStackMap {
+		for stackName, stackManifests := range componentSection {
+			if len(stackManifests) > 1 {
+				m := fmt.Sprintf("the Atmos component '%s' in the stack '%s' is defined in more than one top-level stack manifest file: %s.\n"+
+					"Atmos can't decide which stack manifest to use to get configuration for the component in the stack.\n"+
+					"This is a stack misconfiguration.",
+					componentName,
+					stackName,
+					strings.Join(stackManifests, ", "))
+				res = append(res, m)
+			}
+		}
+	}
+
+	return res, nil
+}