Cleanup tinkerbell test resources before every test run

Makefile

@@ -683,8 +683,10 @@ e2e-tests-binary:
 	GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) test ./test/e2e -c -o "$(E2E_OUTPUT_FILE)" -tags "$(E2E_TAGS)" -ldflags "-X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL)"
 
 .PHONY: build-integration-test-binary
+build-integration-test-binary: ALL_LINKER_FLAGS := $(LINKER_FLAGS) -X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL) -s -w -buildid='' -extldflags -static
+build-integration-test-binary: LINKER_FLAGS_ARG := -ldflags "$(ALL_LINKER_FLAGS)"
 build-integration-test-binary:
-	GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) build -o bin/test github.com/aws/eks-anywhere/cmd/integration_test
+	GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) build $(LINKER_FLAGS_ARG) -o bin/test github.com/aws/eks-anywhere/cmd/integration_test
 
 .PHONY: conformance
 conformance:

cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml

@@ -203,6 +203,12 @@ phases:
         --insecure
         --ignoreErrors
         -v 4
+      - >
+        ./bin/test e2e cleanup tinkerbell
+        --storage-bucket ${INTEGRATION_TEST_STORAGE_BUCKET}
+        --instance-config ${INTEGRATION_TEST_INFRA_CONFIG}
+        --dry-run
+        -v 4
   build:
     commands:
       - export JOB_ID=$CODEBUILD_BUILD_ID

cmd/integration_test/cmd/cleanuptinkerbell.go

@@ -0,0 +1,156 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/spf13/cobra"
+
+	"github.com/aws/eks-anywhere/internal/pkg/ssm"
+	"github.com/aws/eks-anywhere/internal/test/cleanup"
+	"github.com/aws/eks-anywhere/internal/test/e2e"
+	"github.com/aws/eks-anywhere/pkg/dependencies"
+	"github.com/aws/eks-anywhere/pkg/errors"
+	"github.com/aws/eks-anywhere/pkg/executables"
+	"github.com/aws/eks-anywhere/pkg/logger"
+	"github.com/aws/eks-anywhere/pkg/providers/tinkerbell/hardware"
+)
+
+var cleanUpTinkerbellCmd = &cobra.Command{
+	Use:          "tinkerbell",
+	Short:        "Clean up tinkerbell e2e resources",
+	Long:         "Deletes vms created for e2e testing on vsphere and powers off metal machines",
+	SilenceUsage: true,
+	PreRun:       preRunCleanUpNutanixSetup,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return cleanUpTinkerbellTestResources(cmd.Context())
+	},
+}
+
+var (
+	storageBucket  string
+	instanceConfig string
+	dryRun         bool
+)
+
+func init() {
+	cleanUpInstancesCmd.AddCommand(cleanUpTinkerbellCmd)
+	cleanUpTinkerbellCmd.Flags().StringVarP(&storageBucket, storageBucketFlagName, "s", "", "S3 bucket name where tinkerbell hardware inventory files are stored")
+	cleanUpTinkerbellCmd.Flags().StringVar(&instanceConfig, instanceConfigFlagName, "", "File path to the instance-config.yml config")
+	cleanUpTinkerbellCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Run command without deleting or powering off any resources")
+
+	if err := cleanUpTinkerbellCmd.MarkFlagRequired(storageBucketFlagName); err != nil {
+		log.Fatalf("Error marking flag %s as required: %v", storageBucketFlagName, err)
+	}
+
+	if err := cleanUpTinkerbellCmd.MarkFlagRequired(instanceConfigFlagName); err != nil {
+		log.Fatalf("Error marking flag %s as required: %v", instanceConfigFlagName, err)
+	}
+}
+
+// cleanUpTinkerbellTestResources deletes any test runner vm in vsphere and powers off all metal machines.
+func cleanUpTinkerbellTestResources(ctx context.Context) error {
+	session, err := session.NewSession()
+	if err != nil {
+		return fmt.Errorf("creating session: %w", err)
+	}
+
+	deps, err := dependencies.NewFactory().WithGovc().Build(ctx)
+	if err != nil {
+		return err
+	}
+	defer deps.Close(ctx)
+	govc := deps.Govc
+
+	infraConfig, err := e2e.ReadRunnerConfig(instanceConfig)
+	if err != nil {
+		return fmt.Errorf("reading vms config for tests: %v", err)
+	}
+
+	govc.Configure(
+		executables.GovcConfig{
+			Username:   infraConfig.Username,
+			Password:   infraConfig.Password,
+			URL:        infraConfig.URL,
+			Insecure:   infraConfig.Insecure,
+			Datacenter: infraConfig.Datacenter,
+		},
+	)
+
+	var errs []error
+
+	if err := deleteSSMInstances(ctx, session); len(err) != 0 {
+		errs = append(errs, err...)
+	}
+
+	if err := deleteRunners(ctx, govc, infraConfig.Folder); len(err) != 0 {
+		errs = append(errs, err...)
+	}
+
+	if err := powerOffMachines(ctx, session); len(err) != 0 {
+		errs = append(errs, err...)
+	}
+
+	return errors.NewAggregate(errs)
+}
+
+func deleteSSMInstances(ctx context.Context, session *session.Session) []error {
+	var errs []error
+	if ssmInstances, err := e2e.ListTinkerbellSSMInstances(ctx, session); err != nil {
+		errs = append(errs, fmt.Errorf("listing ssm instances: %w", err))
+	} else if dryRun {
+		logger.Info("Found SSM instances", "instanceIDs", ssmInstances.InstanceIDs, "activationIDs", ssmInstances.ActivationIDs)
+	} else {
+		if _, err := ssm.DeregisterInstances(session, ssmInstances.InstanceIDs...); err != nil {
+			errs = append(errs, fmt.Errorf("deleting ssm instances: %w", err))
+		}
+		if _, err := ssm.DeleteActivations(session, ssmInstances.ActivationIDs...); err != nil {
+			errs = append(errs, fmt.Errorf("deleting ssm activations: %w", err))
+		}
+	}
+
+	return errs
+}
+
+func deleteRunners(ctx context.Context, govc *executables.Govc, folder string) []error {
+	var errs []error
+	if runners, err := govc.ListVMs(ctx, folder); err != nil {
+		errs = append(errs, fmt.Errorf("listing tinkerbell runners: %w", err))
+	} else if dryRun {
+		logger.Info("Found VM Runners", "vms", runners)
+	} else {
+		for _, vm := range runners {
+			if err := govc.DeleteVM(ctx, vm.Path); err != nil {
+				errs = append(errs, fmt.Errorf("deleting tinkerbell runner %s: %w", vm, err))
+			}
+		}
+	}
+
+	return errs
+}
+
+func powerOffMachines(_ context.Context, session *session.Session) []error {
+	var errs []error
+	if machines, err := e2e.ReadTinkerbellMachinePool(session, storageBucket); err != nil {
+		errs = append(errs, fmt.Errorf("reading tinkerbell machine pool: %v", err))
+	} else if dryRun {
+		logger.Info("Metal machine pool", "machines", names(machines))
+	} else {
+		if err = cleanup.PowerOffTinkerbellMachines(machines, true); err != nil {
+			errs = append(errs, fmt.Errorf("powering off tinkerbell machines: %v", err))
+		}
+	}
+
+	return errs
+}
+
+func names(h []*hardware.Machine) []string {
+	names := make([]string, 0, len(h))
+	for _, m := range h {
+		names = append(names, m.Hostname)
+	}
+
+	return names
+}

internal/pkg/api/hardware.go

@@ -25,7 +25,7 @@
 // Alias for backwards compatibility.
 type Hardware = hardware.Machine
 
-func NewHardwareSlice(r io.Reader) ([]*Hardware, error) {
+func ReadTinkerbellHardware(r io.Reader) ([]*Hardware, error) {
 	hardware := []*Hardware{}
 
 	if err := gocsv.Unmarshal(r, &hardware); err != nil {
@@ -35,16 +35,16 @@
 	return hardware, nil
 }
 
-func NewHardwareSliceFromFile(file string) ([]*Hardware, error) {
+func ReadTinkerbellHardwareFromFile(file string) ([]*Hardware, error) {
 	hardwareFile, err := os.OpenFile(file, os.O_RDONLY, os.ModePerm)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create hardware slice from hardware file: %v", err)
 	}
-	return NewHardwareSlice(hardwareFile)
+	return ReadTinkerbellHardware(hardwareFile)
 }
 
 func NewHardwareMapFromFile(file string) (map[string]*Hardware, error) {
-	slice, err := NewHardwareSliceFromFile(file)
+	slice, err := ReadTinkerbellHardwareFromFile(file)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create hardware map from hardware file: %v", err)
 	}

internal/pkg/ssm/activation.go

@@ -3,6 +3,7 @@ package ssm
 import (
 	"fmt"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 )
@@ -12,7 +13,14 @@ type ActivationInfo struct {
 	ActivationID   string
 }
 
-func CreateActivation(session *session.Session, instanceName, role string) (*ActivationInfo, error) {
+// Tag is an SSM tag.
+type Tag struct {
+	Key   string
+	Value string
+}
+
+// CreateActivation creates an SSM Hybrid activation.
+func CreateActivation(session *session.Session, instanceName, role string, tags ...Tag) (*ActivationInfo, error) {
 	s := ssm.New(session)
 
 	request := ssm.CreateActivationInput{
@@ -21,6 +29,12 @@ func CreateActivation(session *session.Session, instanceName, role string) (*Act
 		IamRole:             &role,
 	}
 
+	for _, tag := range tags {
+		request.Tags = append(request.Tags,
+			&ssm.Tag{Key: aws.String(tag.Key), Value: aws.String(tag.Value)},
+		)
+	}
+
 	result, err := s.CreateActivation(&request)
 	if err != nil {
 		return nil, fmt.Errorf("failed to activate ssm instance %s: %v", instanceName, err)
@@ -29,17 +43,22 @@ func CreateActivation(session *session.Session, instanceName, role string) (*Act
 	return &ActivationInfo{ActivationCode: *result.ActivationCode, ActivationID: *result.ActivationId}, nil
 }
 
-func DeleteActivation(session *session.Session, activationId string) (*ssm.DeleteActivationOutput, error) {
+// DeleteActivations deletes SSM activations.
+func DeleteActivations(session *session.Session, ids ...string) ([]*ssm.DeleteActivationOutput, error) {
 	s := ssm.New(session)
+	var outputs []*ssm.DeleteActivationOutput
+	for _, id := range ids {
+		request := ssm.DeleteActivationInput{
+			ActivationId: &id,
+		}
 
-	request := ssm.DeleteActivationInput{
-		ActivationId: &activationId,
-	}
+		result, err := s.DeleteActivation(&request)
+		if err != nil {
+			return nil, fmt.Errorf("failed to delete ssm activation: %v", err)
+		}
 
-	result, err := s.DeleteActivation(&request)
-	if err != nil {
-		return nil, fmt.Errorf("failed to delete ssm activation: %v", err)
+		outputs = append(outputs, result)
 	}
 
-	return result, nil
+	return outputs, nil
 }

internal/pkg/ssm/instance.go

@@ -1,8 +1,10 @@
 package ssm
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 )
@@ -30,16 +32,43 @@
 	return infoList[0], nil
 }
 
-func DeregisterInstance(session *session.Session, id string) (*ssm.DeregisterManagedInstanceOutput, error) {
+// DeregisterInstances deregisters SSM instances.
+func DeregisterInstances(session *session.Session, ids ...string) ([]*ssm.DeregisterManagedInstanceOutput, error) {
 	s := ssm.New(session)
-	input := ssm.DeregisterManagedInstanceInput{
-		InstanceId: &id,
+	var outputs []*ssm.DeregisterManagedInstanceOutput
+	for _, id := range ids {
+		input := ssm.DeregisterManagedInstanceInput{
+			InstanceId: &id,
+		}
+
+		output, err := s.DeregisterManagedInstance(&input)
+		if err != nil {
+			return nil, fmt.Errorf("failed to deregister ssm instance %s: %v", id, err)
+		}
+
+		outputs = append(outputs, output)
 	}
 
-	output, err := s.DeregisterManagedInstance(&input)
+	return outputs, nil
+}
+
+func ListInstancesByTags(ctx context.Context, session *session.Session, tags ...Tag) ([]*ssm.InstanceInformation, error) {
+	s := ssm.New(session)
+	input := ssm.DescribeInstanceInformationInput{
+		Filters: make([]*ssm.InstanceInformationStringFilter, 0, len(tags)),
+	}
+
+	for _, tag := range tags {
+		input.Filters = append(input.Filters, &ssm.InstanceInformationStringFilter{
+			Key:    aws.String("tag:" + tag.Key),
+			Values: aws.StringSlice([]string{tag.Value}),
+		})
+	}
+
+	output, err := s.DescribeInstanceInformation(&input)
 	if err != nil {
-		return nil, fmt.Errorf("failed to deregister ssm instance %s: %v", id, err)
+		return nil, fmt.Errorf("listing ssm instances by tags: %v", err)
 	}
 
-	return output, nil
+	return output.InstanceInformationList, nil
 }