From 54a7669d54d17a36e5d24bbc8696f095b0f593ca Mon Sep 17 00:00:00 2001 From: Josh Wolf Date: Thu, 29 Aug 2024 13:45:17 -0400 Subject: [PATCH] change the docker harness to be dind instead of dood --- internal/docker/docker.go | 23 ++- internal/harness/docker/docker.go | 212 ++++++++++++++++++++----- internal/harness/docker/docker_test.go | 43 +++++ internal/harness/docker/opts.go | 20 +-- 4 files changed, 241 insertions(+), 57 deletions(-) create mode 100644 internal/harness/docker/docker_test.go diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 5b9a3d0..9966d14 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -46,6 +46,7 @@ type Request struct { Resources ResourcesRequest Mounts []mount.Mount Networks []NetworkAttachment + NetworkMode string Timeout time.Duration HealthCheck *container.HealthConfig Contents []*Content @@ -153,6 +154,7 @@ func (d *Client) Start(ctx context.Context, req *Request) (*Response, error) { }, Mounts: req.Mounts, PortBindings: req.PortBindings, + NetworkMode: container.NetworkMode(req.NetworkMode), }, &network.NetworkingConfig{ EndpointsConfig: endpointSettings, @@ -382,12 +384,7 @@ func (r *Response) Run(ctx context.Context, cmd harness.Command) error { } func (r *Response) GetFile(ctx context.Context, path string) (io.Reader, error) { - // ensure path is absolute - if !filepath.IsAbs(path) { - return nil, fmt.Errorf("path %s is not absolute", path) - } - - trc, _, err := r.cli.CopyFromContainer(ctx, r.ID, path) + trc, err := r.GetFromContainer(ctx, path) if err != nil { return nil, err } @@ -412,6 +409,20 @@ func (r *Response) GetFile(ctx context.Context, path string) (io.Reader, error) return tr, nil } +func (r *Response) GetFromContainer(ctx context.Context, path string) (io.ReadCloser, error) { + // ensure path is absolute + if !filepath.IsAbs(path) { + return nil, fmt.Errorf("path %s is not absolute", path) + } + + trc, _, err := r.cli.CopyFromContainer(ctx, r.ID, path) + if err != nil { + return nil, err + } + + return trc, nil +} + func (d *Client) withDefaultLabels(labels map[string]string) map[string]string { l := map[string]string{ "dev.chainguard.imagetest": "true", diff --git a/internal/harness/docker/docker.go b/internal/harness/docker/docker.go index af49136..fbfab32 100644 --- a/internal/harness/docker/docker.go +++ b/internal/harness/docker/docker.go @@ -1,28 +1,36 @@ package docker import ( + "archive/tar" + "bytes" "context" "encoding/base64" "encoding/json" "fmt" + "io" + "path/filepath" + "time" - client "github.com/chainguard-dev/terraform-provider-imagetest/internal/docker" + "github.com/chainguard-dev/terraform-provider-imagetest/internal/docker" "github.com/chainguard-dev/terraform-provider-imagetest/internal/harness" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/google/go-containerregistry/pkg/name" "k8s.io/apimachinery/pkg/api/resource" ) -var _ harness.Harness = &docker{} +var _ harness.Harness = &dind{} -const DefaultDockerSocketPath = "/var/run/docker.sock" +const ( + dindCertDir = "/imagetest/certs" +) -type docker struct { +type dind struct { Name string ImageRef name.Reference - Networks []client.NetworkAttachment + Networks []docker.NetworkAttachment Mounts []mount.Mount - Resources client.ResourcesRequest + Resources docker.ResourcesRequest Envs []string Registries map[string]*RegistryConfig Volumes []VolumeConfig @@ -32,9 +40,9 @@ type docker struct { } func New(opts ...Option) (harness.Harness, error) { - h := &docker{ + h := &dind{ ImageRef: name.MustParseReference("cgr.dev/chainguard/docker-cli:latest-dev"), - Resources: client.ResourcesRequest{ + Resources: docker.ResourcesRequest{ MemoryRequest: resource.MustParse("1Gi"), MemoryLimit: resource.MustParse("2Gi"), }, @@ -50,64 +58,132 @@ func New(opts ...Option) (harness.Harness, error) { } } + // translate volumes to mounts + if len(h.Volumes) > 0 { + for _, vol := range h.Volumes { + h.Mounts = append(h.Mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: vol.Name, // mount.Mount refers to "Source" as the name for a named volume + Target: vol.Target, + }) + } + } + return h, nil } // Create implements harness.Harness. -func (h *docker) Create(ctx context.Context) error { - cli, err := client.New() +func (h *dind) Create(ctx context.Context) error { + cli, err := docker.New() if err != nil { return err } - nw, err := cli.CreateNetwork(ctx, &client.NetworkRequest{}) + dresp, err := h.startDaemon(ctx, cli) if err != nil { - return fmt.Errorf("creating network: %w", err) + return fmt.Errorf("starting daemon: %w", err) + } + + if err := h.startSandbox(ctx, cli, dresp); err != nil { + return fmt.Errorf("creating sandbox: %w", err) } + return nil +} + +func (h *dind) startDaemon(ctx context.Context, cli *docker.Client) (*docker.Response, error) { + nw, err := cli.CreateNetwork(ctx, &docker.NetworkRequest{}) + if err != nil { + return nil, fmt.Errorf("creating network: %w", err) + } if err := h.stack.Add(func(ctx context.Context) error { return cli.RemoveNetwork(ctx, nw) }); err != nil { - return fmt.Errorf("adding network teardown to stack: %w", err) + return nil, fmt.Errorf("adding network teardown to stack: %w", err) } + resp, err := cli.Start(ctx, &docker.Request{ + Name: h.Name, + Ref: name.MustParseReference("docker:dind"), + Privileged: true, + Networks: append(h.Networks, docker.NetworkAttachment{ + Name: nw.Name, + ID: nw.ID, + }), + Env: []string{ + fmt.Sprintf("DOCKER_TLS_CERTDIR=%s", dindCertDir), + }, + HealthCheck: &container.HealthConfig{ + Test: []string{"CMD", "/bin/sh", "-c", "docker info"}, + Interval: 1 * time.Second, + Timeout: 5 * time.Second, + Retries: 5, + StartPeriod: 1 * time.Second, + }, + Mounts: h.Mounts, + }) + if err != nil { + return nil, fmt.Errorf("starting container: %w", err) + } + + if err := h.stack.Add(func(ctx context.Context) error { + return cli.Remove(ctx, resp) + }); err != nil { + return nil, fmt.Errorf("adding container teardown to stack: %w", err) + } + + return resp, nil +} + +func (h *dind) startSandbox(ctx context.Context, cli *docker.Client, dresp *docker.Response) error { dockerconfigjson, err := createDockerConfigJSON(h.Registries) if err != nil { return fmt.Errorf("creating docker config json: %w", err) } - mounts := append(h.Mounts, mount.Mount{ - Type: mount.TypeBind, - Source: "/var/run/docker.sock", - Target: "/var/run/docker.sock", - }) + nws := make(map[string]struct{}) - if len(h.Volumes) > 0 { - for _, vol := range h.Volumes { - mounts = append(mounts, mount.Mount{ - Type: mount.TypeVolume, - Source: vol.Name, // mount.Mount refers to "Source" as the name for a named volume - Target: vol.Target, + // Attach the sandbox to any networks k3s is also a part of, excluding any + // invalid networks or networks already attached (the daemon cannot deconflict + // these) + for nn, nw := range dresp.NetworkSettings.Networks { + if nn == "" { + continue + } + if _, ok := nws[nn]; !ok { + nws[nn] = struct{}{} + h.Networks = append(h.Networks, docker.NetworkAttachment{ + Name: nn, + ID: nw.NetworkID, }) } } - resp, err := cli.Start(ctx, &client.Request{ - Name: h.Name, + certs, err := h.certContents(ctx, dresp) + if err != nil { + return fmt.Errorf("getting certs from dind: %w", err) + } + + name := dresp.Name + "-sandbox" + + resp, err := cli.Start(ctx, &docker.Request{ + Name: name, Ref: h.ImageRef, Entrypoint: harness.DefaultEntrypoint(), Cmd: harness.DefaultCmd(), Networks: h.Networks, Resources: h.Resources, User: "0:0", - Mounts: mounts, - Env: h.Envs, - Contents: []*client.Content{ - client.NewContentFromString(string(dockerconfigjson), "/root/.docker/config.json"), - }, - ExtraHosts: []string{ - "host.docker.internal:host-gateway", - }, + Env: append(h.Envs, + fmt.Sprintf("DOCKER_HOST=tcp://%s:2376", dresp.Config.Hostname), + "DOCKER_TLS_VERIFY=1", + fmt.Sprintf("DOCKER_CERT_PATH=%s", filepath.Join(dindCertDir, "client")), + ), + Mounts: h.Mounts, + Contents: append([]*docker.Content{ + docker.NewContentFromString(string(dockerconfigjson), "/root/.docker/config.json"), + }, certs...), + NetworkMode: fmt.Sprintf("container:%s", dresp.ID), }) if err != nil { return fmt.Errorf("starting container: %w", err) @@ -127,16 +203,11 @@ func (h *docker) Create(ctx context.Context) error { } // Run implements harness.Harness. -func (h *docker) Run(ctx context.Context, cmd harness.Command) error { +func (h *dind) Run(ctx context.Context, cmd harness.Command) error { return h.runner(ctx, cmd) } -func (h *docker) DebugLogCommand() string { - // TODO implement something here - return "" -} - -func (h *docker) Destroy(ctx context.Context) error { +func (h *dind) Destroy(ctx context.Context) error { return h.stack.Teardown(ctx) } @@ -176,3 +247,62 @@ func createDockerConfigJSON(registryAuths map[string]*RegistryConfig) ([]byte, e return dockerConfigJSON, nil } + +// certContents grabs the certs from the dind container and returns them as +// content. We could use a mount here, but we're trying to get away from bind +// mounts and realistically we're only inefficiently copying 3 very small files +// here. +func (h *dind) certContents(ctx context.Context, resp *docker.Response) ([]*docker.Content, error) { + rc, err := resp.GetFromContainer(ctx, filepath.Join(dindCertDir, "client")) + if err != nil { + return nil, fmt.Errorf("getting certs from container: %w", err) + } + + tr := tar.NewReader(rc) + + contents := map[string]*docker.Content{ + "ca.pem": nil, + "cert.pem": nil, + "key.pem": nil, + } + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("reading certs from container: %w", err) + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + name := filepath.Base(hdr.Name) + + switch name { + case "ca.pem", "cert.pem", "key.pem": + var buf bytes.Buffer + if _, err := io.Copy(&buf, tr); err != nil { + return nil, fmt.Errorf("reading certs from container: %w", err) + } + + contents[name] = docker.NewContentFromString(buf.String(), filepath.Join("/imagetest/certs/client", name)) + } + } + + if err := rc.Close(); err != nil { + return nil, fmt.Errorf("closing certs reader: %w", err) + } + + c := []*docker.Content{} + for k, v := range contents { + if v == nil { + return nil, fmt.Errorf("no %s found in dind container", k) + } + c = append(c, v) + } + + return c, nil +} diff --git a/internal/harness/docker/docker_test.go b/internal/harness/docker/docker_test.go new file mode 100644 index 0000000..151a5df --- /dev/null +++ b/internal/harness/docker/docker_test.go @@ -0,0 +1,43 @@ +package docker + +import ( + "context" + "testing" + + "github.com/chainguard-dev/terraform-provider-imagetest/internal/harness" + "github.com/stretchr/testify/require" +) + +func TestDocker(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := context.Background() + + d, err := New() + require.NoError(t, err) + require.NotNil(t, d) + + // Create the harness + err = d.Create(ctx) + require.NoError(t, err) + + // Ensure we can use docker + err = d.Run(ctx, harness.Command{ + Args: "docker run --rm hello-world", + }) + require.NoError(t, err) + + // Ensure we can start a container and hit it via localhost + err = d.Run(ctx, harness.Command{ + Args: "docker run -d --rm -p 8080:80 nginx && apk add curl && curl -v http://localhost:8080", + }) + require.NoError(t, err) + + // Run a command that should fail + err = d.Run(ctx, harness.Command{ + Args: "exit 1", + }) + require.ErrorContains(t, err, "exit 1") +} diff --git a/internal/harness/docker/opts.go b/internal/harness/docker/opts.go index a6c2f94..1c010c3 100644 --- a/internal/harness/docker/opts.go +++ b/internal/harness/docker/opts.go @@ -9,7 +9,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" ) -type Option func(*docker) error +type Option func(*dind) error type VolumeConfig struct { Name string @@ -34,21 +34,21 @@ type RegistryTlsConfig struct { } func WithName(name string) Option { - return func(opt *docker) error { + return func(opt *dind) error { opt.Name = name return nil } } func WithImageRef(ref name.Reference) Option { - return func(opt *docker) error { + return func(opt *dind) error { opt.ImageRef = ref return nil } } func WithMounts(mounts ...mount.Mount) Option { - return func(opt *docker) error { + return func(opt *dind) error { if mounts != nil { opt.Mounts = append(opt.Mounts, mounts...) } @@ -57,14 +57,14 @@ func WithMounts(mounts ...mount.Mount) Option { } func WithNetworks(networks ...client.NetworkAttachment) Option { - return func(opt *docker) error { + return func(opt *dind) error { opt.Networks = append(opt.Networks, networks...) return nil } } func WithAuthFromStatic(registry, username, password, auth string) Option { - return func(opt *docker) error { + return func(opt *dind) error { if opt.Registries == nil { opt.Registries = make(map[string]*RegistryConfig) } @@ -83,7 +83,7 @@ func WithAuthFromStatic(registry, username, password, auth string) Option { } func WithAuthFromKeychain(registry string) Option { - return func(opt *docker) error { + return func(opt *dind) error { if opt.Registries == nil { opt.Registries = make(map[string]*RegistryConfig) } @@ -117,7 +117,7 @@ func WithAuthFromKeychain(registry string) Option { } func WithEnvs(env ...string) Option { - return func(opt *docker) error { + return func(opt *dind) error { if opt.Envs == nil { opt.Envs = make([]string, 0) } @@ -127,14 +127,14 @@ func WithEnvs(env ...string) Option { } func WithResources(req client.ResourcesRequest) Option { - return func(opt *docker) error { + return func(opt *dind) error { opt.Resources = req return nil } } func WithVolumes(volumes ...VolumeConfig) Option { - return func(opt *docker) error { + return func(opt *dind) error { if volumes == nil { return nil }