change the docker harness to be dind instead of dood

internal/docker/docker.go

@@ -46,6 +46,7 @@ type Request struct {
 	Resources    ResourcesRequest
 	Mounts       []mount.Mount
 	Networks     []NetworkAttachment
+	NetworkMode  string
 	Timeout      time.Duration
 	HealthCheck  *container.HealthConfig
 	Contents     []*Content
@@ -153,6 +154,7 @@ func (d *Client) Start(ctx context.Context, req *Request) (*Response, error) {
 			},
 			Mounts:       req.Mounts,
 			PortBindings: req.PortBindings,
+			NetworkMode:  container.NetworkMode(req.NetworkMode),
 		},
 		&network.NetworkingConfig{
 			EndpointsConfig: endpointSettings,
@@ -382,12 +384,7 @@ func (r *Response) Run(ctx context.Context, cmd harness.Command) error {
 }
 
 func (r *Response) GetFile(ctx context.Context, path string) (io.Reader, error) {
-	// ensure path is absolute
-	if !filepath.IsAbs(path) {
-		return nil, fmt.Errorf("path %s is not absolute", path)
-	}
-
-	trc, _, err := r.cli.CopyFromContainer(ctx, r.ID, path)
+	trc, err := r.GetFromContainer(ctx, path)
 	if err != nil {
 		return nil, err
 	}
@@ -412,6 +409,20 @@ func (r *Response) GetFile(ctx context.Context, path string) (io.Reader, error)
 	return tr, nil
 }
 
+func (r *Response) GetFromContainer(ctx context.Context, path string) (io.ReadCloser, error) {
+	// ensure path is absolute
+	if !filepath.IsAbs(path) {
+		return nil, fmt.Errorf("path %s is not absolute", path)
+	}
+
+	trc, _, err := r.cli.CopyFromContainer(ctx, r.ID, path)
+	if err != nil {
+		return nil, err
+	}
+
+	return trc, nil
+}
+
 func (d *Client) withDefaultLabels(labels map[string]string) map[string]string {
 	l := map[string]string{
 		"dev.chainguard.imagetest": "true",

internal/harness/docker/docker.go

@@ -1,28 +1,36 @@
 package docker
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
+	"path/filepath"
+	"time"
 
-	client "github.com/chainguard-dev/terraform-provider-imagetest/internal/docker"
+	"github.com/chainguard-dev/terraform-provider-imagetest/internal/docker"
 	"github.com/chainguard-dev/terraform-provider-imagetest/internal/harness"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/google/go-containerregistry/pkg/name"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
-var _ harness.Harness = &docker{}
+var _ harness.Harness = &dind{}
 
-const DefaultDockerSocketPath = "/var/run/docker.sock"
+const (
+	dindCertDir = "/imagetest/certs"
+)
 
-type docker struct {
+type dind struct {
 	Name       string
 	ImageRef   name.Reference
-	Networks   []client.NetworkAttachment
+	Networks   []docker.NetworkAttachment
 	Mounts     []mount.Mount
-	Resources  client.ResourcesRequest
+	Resources  docker.ResourcesRequest
 	Envs       []string
 	Registries map[string]*RegistryConfig
 	Volumes    []VolumeConfig
@@ -32,9 +40,9 @@ type docker struct {
 }
 
 func New(opts ...Option) (harness.Harness, error) {
-	h := &docker{
+	h := &dind{
 		ImageRef: name.MustParseReference("cgr.dev/chainguard/docker-cli:latest-dev"),
-		Resources: client.ResourcesRequest{
+		Resources: docker.ResourcesRequest{
 			MemoryRequest: resource.MustParse("1Gi"),
 			MemoryLimit:   resource.MustParse("2Gi"),
 		},
@@ -50,64 +58,132 @@ func New(opts ...Option) (harness.Harness, error) {
 		}
 	}
 
+	// translate volumes to mounts
+	if len(h.Volumes) > 0 {
+		for _, vol := range h.Volumes {
+			h.Mounts = append(h.Mounts, mount.Mount{
+				Type:   mount.TypeVolume,
+				Source: vol.Name, // mount.Mount refers to "Source" as the name for a named volume
+				Target: vol.Target,
+			})
+		}
+	}
+
 	return h, nil
 }
 
 // Create implements harness.Harness.
-func (h *docker) Create(ctx context.Context) error {
-	cli, err := client.New()
+func (h *dind) Create(ctx context.Context) error {
+	cli, err := docker.New()
 	if err != nil {
 		return err
 	}
 
-	nw, err := cli.CreateNetwork(ctx, &client.NetworkRequest{})
+	dresp, err := h.startDaemon(ctx, cli)
 	if err != nil {
-		return fmt.Errorf("creating network: %w", err)
+		return fmt.Errorf("starting daemon: %w", err)
+	}
+
+	if err := h.startSandbox(ctx, cli, dresp); err != nil {
+		return fmt.Errorf("creating sandbox: %w", err)
 	}
 
+	return nil
+}
+
+func (h *dind) startDaemon(ctx context.Context, cli *docker.Client) (*docker.Response, error) {
+	nw, err := cli.CreateNetwork(ctx, &docker.NetworkRequest{})
+	if err != nil {
+		return nil, fmt.Errorf("creating network: %w", err)
+	}
 	if err := h.stack.Add(func(ctx context.Context) error {
 		return cli.RemoveNetwork(ctx, nw)
 	}); err != nil {
-		return fmt.Errorf("adding network teardown to stack: %w", err)
+		return nil, fmt.Errorf("adding network teardown to stack: %w", err)
 	}
 
+	resp, err := cli.Start(ctx, &docker.Request{
+		Name:       h.Name,
+		Ref:        name.MustParseReference("docker:dind"),
+		Privileged: true,
+		Networks: append(h.Networks, docker.NetworkAttachment{
+			Name: nw.Name,
+			ID:   nw.ID,
+		}),
+		Env: []string{
+			fmt.Sprintf("DOCKER_TLS_CERTDIR=%s", dindCertDir),
+		},
+		HealthCheck: &container.HealthConfig{
+			Test:        []string{"CMD", "/bin/sh", "-c", "docker info"},
+			Interval:    1 * time.Second,
+			Timeout:     5 * time.Second,
+			Retries:     5,
+			StartPeriod: 1 * time.Second,
+		},
+		Mounts: h.Mounts,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("starting container: %w", err)
+	}
+
+	if err := h.stack.Add(func(ctx context.Context) error {
+		return cli.Remove(ctx, resp)
+	}); err != nil {
+		return nil, fmt.Errorf("adding container teardown to stack: %w", err)
+	}
+
+	return resp, nil
+}
+
+func (h *dind) startSandbox(ctx context.Context, cli *docker.Client, dresp *docker.Response) error {
 	dockerconfigjson, err := createDockerConfigJSON(h.Registries)
 	if err != nil {
 		return fmt.Errorf("creating docker config json: %w", err)
 	}
 
-	mounts := append(h.Mounts, mount.Mount{
-		Type:   mount.TypeBind,
-		Source: "/var/run/docker.sock",
-		Target: "/var/run/docker.sock",
-	})
+	nws := make(map[string]struct{})
 
-	if len(h.Volumes) > 0 {
-		for _, vol := range h.Volumes {
-			mounts = append(mounts, mount.Mount{
-				Type:   mount.TypeVolume,
-				Source: vol.Name, // mount.Mount refers to "Source" as the name for a named volume
-				Target: vol.Target,
+	// Attach the sandbox to any networks k3s is also a part of, excluding any
+	// invalid networks or networks already attached (the daemon cannot deconflict
+	// these)
+	for nn, nw := range dresp.NetworkSettings.Networks {
+		if nn == "" {
+			continue
+		}
+		if _, ok := nws[nn]; !ok {
+			nws[nn] = struct{}{}
+			h.Networks = append(h.Networks, docker.NetworkAttachment{
+				Name: nn,
+				ID:   nw.NetworkID,
 			})
 		}
 	}
 
-	resp, err := cli.Start(ctx, &client.Request{
-		Name:       h.Name,
+	certs, err := h.certContents(ctx, dresp)
+	if err != nil {
+		return fmt.Errorf("getting certs from dind: %w", err)
+	}
+
+	name := dresp.Name + "-sandbox"
+
+	resp, err := cli.Start(ctx, &docker.Request{
+		Name:       name,
 		Ref:        h.ImageRef,
 		Entrypoint: harness.DefaultEntrypoint(),
 		Cmd:        harness.DefaultCmd(),
 		Networks:   h.Networks,
 		Resources:  h.Resources,
 		User:       "0:0",
-		Mounts:     mounts,
-		Env:        h.Envs,
-		Contents: []*client.Content{
-			client.NewContentFromString(string(dockerconfigjson), "/root/.docker/config.json"),
-		},
-		ExtraHosts: []string{
-			"host.docker.internal:host-gateway",
-		},
+		Env: append(h.Envs,
+			fmt.Sprintf("DOCKER_HOST=tcp://%s:2376", dresp.Config.Hostname),
+			"DOCKER_TLS_VERIFY=1",
+			fmt.Sprintf("DOCKER_CERT_PATH=%s", filepath.Join(dindCertDir, "client")),
+		),
+		Mounts: h.Mounts,
+		Contents: append([]*docker.Content{
+			docker.NewContentFromString(string(dockerconfigjson), "/root/.docker/config.json"),
+		}, certs...),
+		NetworkMode: fmt.Sprintf("container:%s", dresp.ID),
 	})
 	if err != nil {
 		return fmt.Errorf("starting container: %w", err)
@@ -127,16 +203,11 @@ func (h *docker) Create(ctx context.Context) error {
 }
 
 // Run implements harness.Harness.
-func (h *docker) Run(ctx context.Context, cmd harness.Command) error {
+func (h *dind) Run(ctx context.Context, cmd harness.Command) error {
 	return h.runner(ctx, cmd)
 }
 
-func (h *docker) DebugLogCommand() string {
-	// TODO implement something here
-	return ""
-}
-
-func (h *docker) Destroy(ctx context.Context) error {
+func (h *dind) Destroy(ctx context.Context) error {
 	return h.stack.Teardown(ctx)
 }
 
@@ -176,3 +247,62 @@ func createDockerConfigJSON(registryAuths map[string]*RegistryConfig) ([]byte, e
 
 	return dockerConfigJSON, nil
 }
+
+// certContents grabs the certs from the dind container and returns them as
+// content. We could use a mount here, but we're trying to get away from bind
+// mounts and realistically we're only inefficiently copying 3 very small files
+// here.
+func (h *dind) certContents(ctx context.Context, resp *docker.Response) ([]*docker.Content, error) {
+	rc, err := resp.GetFromContainer(ctx, filepath.Join(dindCertDir, "client"))
+	if err != nil {
+		return nil, fmt.Errorf("getting certs from container: %w", err)
+	}
+
+	tr := tar.NewReader(rc)
+
+	contents := map[string]*docker.Content{
+		"ca.pem":   nil,
+		"cert.pem": nil,
+		"key.pem":  nil,
+	}
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("reading certs from container: %w", err)
+		}
+
+		if hdr.Typeflag != tar.TypeReg {
+			continue
+		}
+
+		name := filepath.Base(hdr.Name)
+
+		switch name {
+		case "ca.pem", "cert.pem", "key.pem":
+			var buf bytes.Buffer
+			if _, err := io.Copy(&buf, tr); err != nil {
+				return nil, fmt.Errorf("reading certs from container: %w", err)
+			}
+
+			contents[name] = docker.NewContentFromString(buf.String(), filepath.Join("/imagetest/certs/client", name))
+		}
+	}
+
+	if err := rc.Close(); err != nil {
+		return nil, fmt.Errorf("closing certs reader: %w", err)
+	}
+
+	c := []*docker.Content{}
+	for k, v := range contents {
+		if v == nil {
+			return nil, fmt.Errorf("no %s found in dind container", k)
+		}
+		c = append(c, v)
+	}
+
+	return c, nil
+}

internal/harness/docker/docker_test.go

@@ -0,0 +1,43 @@
+package docker
+
+import (
+	"context"
+	"testing"
+
+	"github.com/chainguard-dev/terraform-provider-imagetest/internal/harness"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDocker(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	ctx := context.Background()
+
+	d, err := New()
+	require.NoError(t, err)
+	require.NotNil(t, d)
+
+	// Create the harness
+	err = d.Create(ctx)
+	require.NoError(t, err)
+
+	// Ensure we can use docker
+	err = d.Run(ctx, harness.Command{
+		Args: "docker run --rm hello-world",
+	})
+	require.NoError(t, err)
+
+	// Ensure we can start a container and hit it via localhost
+	err = d.Run(ctx, harness.Command{
+		Args: "docker run -d --rm -p 8080:80 nginx && apk add curl && curl -v http://localhost:8080",
+	})
+	require.NoError(t, err)
+
+	// Run a command that should fail
+	err = d.Run(ctx, harness.Command{
+		Args: "exit 1",
+	})
+	require.ErrorContains(t, err, "exit 1")
+}