From 135cdd8d34db3883b0b3ef9d2cd14537b222e5da Mon Sep 17 00:00:00 2001
From: Andrew Mains <amains@uber.com>
Date: Fri, 2 Sep 2022 10:50:29 -0400
Subject: [PATCH 1/2] etcd_docker 4: Incorporate docker based etcd into Go
 integration tests (#4148)

PR 4 for https://github.com/m3db/m3/issues/4144

High level approach is as described in https://github.com/m3db/m3/issues/4144 .

This PR integrates docker based etcd into our Go integration tests. It removes the need to have the embed package running
in m3db for them, but doesn't yet touch that functionality.

commit-id:3ae12ffd
---
 src/integration/aggregator/aggregator.go      |  11 ++
 src/integration/aggregator/aggregator_test.go |   2 +
 .../repair/repair_and_replication_test.go     |  74 +++++---
 .../resources/coordinator_client.go           |  50 +++---
 .../resources/docker/dockerexternal/etcd.go   |   2 -
 .../resources/inprocess/aggregator.go         |  14 ++
 .../resources/inprocess/cluster.go            | 166 ++++++++++--------
 .../resources/inprocess/coordinator_test.go   |  16 ++
 .../resources/inprocess/dbnode_test.go        |   2 +
 .../resources/inprocess/inprocess.go          |  11 ++
 src/integration/resources/options.go          |  23 +++
 11 files changed, 254 insertions(+), 117 deletions(-)

diff --git a/src/integration/aggregator/aggregator.go b/src/integration/aggregator/aggregator.go
index 5e2dd0878f..c1fad2922e 100644
--- a/src/integration/aggregator/aggregator.go
+++ b/src/integration/aggregator/aggregator.go
@@ -117,6 +117,17 @@ ingest:
         maxBackoff: 10s
         jitter: true
 storeMetricsType: true
+
+clusterManagement:
+  etcd:
+    env: default_env
+    zone: embedded
+    service: m3db
+    cacheDir: /var/lib/m3kv
+    etcdClusters:
+    - zone: embedded
+      endpoints:
+      - 127.0.0.1:2379
 `
 
 	// TestAggregatorAggregatorConfig is the test config for the aggregators.
diff --git a/src/integration/aggregator/aggregator_test.go b/src/integration/aggregator/aggregator_test.go
index 91bf35082f..0e2d48a289 100644
--- a/src/integration/aggregator/aggregator_test.go
+++ b/src/integration/aggregator/aggregator_test.go
@@ -1,4 +1,6 @@
+//go:build cluster_integration
 // +build cluster_integration
+
 //
 // Copyright (c) 2021  Uber Technologies, Inc.
 //
diff --git a/src/integration/repair/repair_and_replication_test.go b/src/integration/repair/repair_and_replication_test.go
index d122650225..2e88acfb8d 100644
--- a/src/integration/repair/repair_and_replication_test.go
+++ b/src/integration/repair/repair_and_replication_test.go
@@ -1,4 +1,6 @@
+//go:build cluster_integration
 // +build cluster_integration
+
 //
 // Copyright (c) 2021  Uber Technologies, Inc.
 //
@@ -23,16 +25,21 @@
 package repair
 
 import (
+	"context"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
 	"github.com/m3db/m3/src/integration/resources"
+	"github.com/m3db/m3/src/integration/resources/docker/dockerexternal"
 	"github.com/m3db/m3/src/integration/resources/inprocess"
+	"github.com/m3db/m3/src/x/instrument"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestRepairAndReplication(t *testing.T) {
+	t.Skip("failing after etcd containerization; fix.")
 	cluster1, cluster2, closer := testSetup(t)
 	defer closer()
 
@@ -40,11 +47,23 @@ func TestRepairAndReplication(t *testing.T) {
 }
 
 func testSetup(t *testing.T) (resources.M3Resources, resources.M3Resources, func()) {
-	fullCfgs1 := getClusterFullConfgs(t)
-	fullCfgs2 := getClusterFullConfgs(t)
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err)
 
-	ep1 := fullCfgs1.Configs.Coordinator.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0].Endpoints
-	ep2 := fullCfgs2.Configs.Coordinator.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0].Endpoints
+	etcd1 := mustNewStartedEtcd(t, pool)
+	etcd2 := mustNewStartedEtcd(t, pool)
+
+	ep1 := []string{etcd1.Address()}
+	ep2 := []string{etcd2.Address()}
+
+	cluster1Opts := newTestClusterOptions()
+	cluster1Opts.EtcdEndpoints = ep1
+
+	cluster2Opts := newTestClusterOptions()
+	cluster2Opts.EtcdEndpoints = ep2
+
+	fullCfgs1 := getClusterFullConfgs(t, cluster1Opts)
+	fullCfgs2 := getClusterFullConfgs(t, cluster2Opts)
 
 	setRepairAndReplicationCfg(
 		&fullCfgs1,
@@ -57,19 +76,28 @@ func testSetup(t *testing.T) (resources.M3Resources, resources.M3Resources, func
 		ep1,
 	)
 
-	cluster1, err := inprocess.NewClusterFromSpecification(fullCfgs1, clusterOptions)
+	cluster1, err := inprocess.NewClusterFromSpecification(fullCfgs1, cluster1Opts)
 	require.NoError(t, err)
 
-	cluster2, err := inprocess.NewClusterFromSpecification(fullCfgs2, clusterOptions)
+	cluster2, err := inprocess.NewClusterFromSpecification(fullCfgs2, cluster2Opts)
 	require.NoError(t, err)
 
 	return cluster1, cluster2, func() {
+		etcd1.Close(context.TODO())
+		etcd2.Close(context.TODO())
 		assert.NoError(t, cluster1.Cleanup())
 		assert.NoError(t, cluster2.Cleanup())
 	}
 }
 
-func getClusterFullConfgs(t *testing.T) inprocess.ClusterSpecification {
+func mustNewStartedEtcd(t *testing.T, pool *dockertest.Pool) *dockerexternal.EtcdNode {
+	etcd, err := dockerexternal.NewEtcd(pool, instrument.NewOptions())
+	require.NoError(t, err)
+	require.NoError(t, etcd.Setup(context.TODO()))
+	return etcd
+}
+
+func getClusterFullConfgs(t *testing.T, clusterOptions resources.ClusterOptions) inprocess.ClusterSpecification {
 	cfgs, err := inprocess.NewClusterConfigsFromYAML(
 		TestRepairDBNodeConfig, TestRepairCoordinatorConfig, "",
 	)
@@ -84,18 +112,22 @@ func getClusterFullConfgs(t *testing.T) inprocess.ClusterSpecification {
 func setRepairAndReplicationCfg(fullCfg *inprocess.ClusterSpecification, clusterName string, endpoints []string) {
 	for _, dbnode := range fullCfg.Configs.DBNodes {
 		dbnode.DB.Replication.Clusters[0].Name = clusterName
-		dbnode.DB.Replication.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0].Endpoints = endpoints
+		etcdService := &(dbnode.DB.Replication.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0])
+		etcdService.AutoSyncInterval = -1
+		etcdService.Endpoints = endpoints
 	}
 }
 
-var clusterOptions = resources.ClusterOptions{
-	DBNode: &resources.DBNodeClusterOptions{
-		RF:                 2,
-		NumShards:          4,
-		NumInstances:       1,
-		NumIsolationGroups: 2,
-	},
-	Coordinator: resources.CoordinatorClusterOptions{
-		GeneratePorts: true,
-	},
+func newTestClusterOptions() resources.ClusterOptions {
+	return resources.ClusterOptions{
+		DBNode: &resources.DBNodeClusterOptions{
+			RF:                 2,
+			NumShards:          4,
+			NumInstances:       1,
+			NumIsolationGroups: 2,
+		},
+		Coordinator: resources.CoordinatorClusterOptions{
+			GeneratePorts: true,
+		},
+	}
 }
diff --git a/src/integration/resources/coordinator_client.go b/src/integration/resources/coordinator_client.go
index b930fbc52b..72a0dc74d3 100644
--- a/src/integration/resources/coordinator_client.go
+++ b/src/integration/resources/coordinator_client.go
@@ -59,8 +59,8 @@ var errUnknownServiceType = errors.New("unknown service type")
 // operation until successful.
 type RetryFunc func(op func() error) error
 
-// ZapMethod appends the method as a log field.
-func ZapMethod(s string) zapcore.Field { return zap.String("method", s) }
+// zapMethod appends the method as a log field.
+func zapMethod(s string) zapcore.Field { return zap.String("method", s) }
 
 // CoordinatorClient is a client use to invoke API calls
 // on a coordinator
@@ -97,7 +97,7 @@ func (c *CoordinatorClient) makeURL(resource string) string {
 func (c *CoordinatorClient) GetNamespace() (admin.NamespaceGetResponse, error) {
 	url := c.makeURL("api/v1/services/m3db/namespace")
 	logger := c.logger.With(
-		ZapMethod("getNamespace"), zap.String("url", url))
+		zapMethod("getNamespace"), zap.String("url", url))
 
 	//nolint:noctx
 	resp, err := c.client.Get(url)
@@ -129,7 +129,7 @@ func (c *CoordinatorClient) GetPlacement(opts PlacementRequestOptions) (admin.Pl
 	}
 	url := c.makeURL(handlerurl)
 	logger := c.logger.With(
-		ZapMethod("getPlacement"), zap.String("url", url))
+		zapMethod("getPlacement"), zap.String("url", url))
 
 	resp, err := c.makeRequest(logger, url, placementhandler.GetHTTPMethod, nil, placementOptsToMap(opts))
 	if err != nil {
@@ -163,7 +163,7 @@ func (c *CoordinatorClient) InitPlacement(
 	}
 	url := c.makeURL(handlerurl)
 	logger := c.logger.With(
-		ZapMethod("initPlacement"), zap.String("url", url))
+		zapMethod("initPlacement"), zap.String("url", url))
 
 	resp, err := c.makeRequest(logger, url, placementhandler.InitHTTPMethod, &initRequest, placementOptsToMap(opts))
 	if err != nil {
@@ -194,7 +194,7 @@ func (c *CoordinatorClient) DeleteAllPlacements(opts PlacementRequestOptions) er
 	}
 	url := c.makeURL(handlerurl)
 	logger := c.logger.With(
-		ZapMethod("deleteAllPlacements"), zap.String("url", url))
+		zapMethod("deleteAllPlacements"), zap.String("url", url))
 
 	resp, err := c.makeRequest(
 		logger, url, placementhandler.DeleteAllHTTPMethod, nil, placementOptsToMap(opts),
@@ -221,7 +221,7 @@ func (c *CoordinatorClient) DeleteAllPlacements(opts PlacementRequestOptions) er
 // NB: if the name string is empty, this will instead
 // check for a successful response.
 func (c *CoordinatorClient) WaitForNamespace(name string) error {
-	logger := c.logger.With(ZapMethod("waitForNamespace"))
+	logger := c.logger.With(zapMethod("waitForNamespace"))
 	return c.retryFunc(func() error {
 		ns, err := c.GetNamespace()
 		if err != nil {
@@ -250,7 +250,7 @@ func (c *CoordinatorClient) WaitForNamespace(name string) error {
 func (c *CoordinatorClient) WaitForInstances(
 	ids []string,
 ) error {
-	logger := c.logger.With(ZapMethod("waitForPlacement"))
+	logger := c.logger.With(zapMethod("waitForPlacement"))
 	return c.retryFunc(func() error {
 		placement, err := c.GetPlacement(PlacementRequestOptions{Service: ServiceTypeM3DB})
 		if err != nil {
@@ -282,7 +282,7 @@ func (c *CoordinatorClient) WaitForInstances(
 
 // WaitForShardsReady waits until all shards gets ready.
 func (c *CoordinatorClient) WaitForShardsReady() error {
-	logger := c.logger.With(ZapMethod("waitForShards"))
+	logger := c.logger.With(zapMethod("waitForShards"))
 	return c.retryFunc(func() error {
 		placement, err := c.GetPlacement(PlacementRequestOptions{Service: ServiceTypeM3DB})
 		if err != nil {
@@ -307,7 +307,7 @@ func (c *CoordinatorClient) WaitForShardsReady() error {
 func (c *CoordinatorClient) WaitForClusterReady() error {
 	var (
 		url    = c.makeURL("ready")
-		logger = c.logger.With(ZapMethod("waitForClusterReady"), zap.String("url", url))
+		logger = c.logger.With(zapMethod("waitForClusterReady"), zap.String("url", url))
 	)
 	return c.retryFunc(func() error {
 		req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
@@ -350,7 +350,7 @@ func (c *CoordinatorClient) CreateDatabase(
 ) (admin.DatabaseCreateResponse, error) {
 	url := c.makeURL("api/v1/database/create")
 	logger := c.logger.With(
-		ZapMethod("createDatabase"), zap.String("url", url),
+		zapMethod("createDatabase"), zap.String("url", url),
 		zap.String("request", addRequest.String()))
 
 	resp, err := c.makeRequest(logger, url, http.MethodPost, &addRequest, nil)
@@ -383,7 +383,7 @@ func (c *CoordinatorClient) AddNamespace(
 ) (admin.NamespaceGetResponse, error) {
 	url := c.makeURL("api/v1/services/m3db/namespace")
 	logger := c.logger.With(
-		ZapMethod("addNamespace"), zap.String("url", url),
+		zapMethod("addNamespace"), zap.String("url", url),
 		zap.String("request", addRequest.String()))
 
 	resp, err := c.makeRequest(logger, url, http.MethodPost, &addRequest, nil)
@@ -411,7 +411,7 @@ func (c *CoordinatorClient) UpdateNamespace(
 ) (admin.NamespaceGetResponse, error) {
 	url := c.makeURL("api/v1/services/m3db/namespace")
 	logger := c.logger.With(
-		ZapMethod("updateNamespace"), zap.String("url", url),
+		zapMethod("updateNamespace"), zap.String("url", url),
 		zap.String("request", req.String()))
 
 	resp, err := c.makeRequest(logger, url, http.MethodPut, &req, nil)
@@ -431,7 +431,7 @@ func (c *CoordinatorClient) UpdateNamespace(
 func (c *CoordinatorClient) setNamespaceReady(name string) error {
 	url := c.makeURL("api/v1/services/m3db/namespace/ready")
 	logger := c.logger.With(
-		ZapMethod("setNamespaceReady"), zap.String("url", url),
+		zapMethod("setNamespaceReady"), zap.String("url", url),
 		zap.String("namespace", name))
 
 	_, err := c.makeRequest(logger, url, http.MethodPost, // nolint: bodyclose
@@ -445,7 +445,7 @@ func (c *CoordinatorClient) setNamespaceReady(name string) error {
 // DeleteNamespace removes the namespace.
 func (c *CoordinatorClient) DeleteNamespace(namespaceID string) error {
 	url := c.makeURL("api/v1/services/m3db/namespace/" + namespaceID)
-	logger := c.logger.With(ZapMethod("deleteNamespace"), zap.String("url", url))
+	logger := c.logger.With(zapMethod("deleteNamespace"), zap.String("url", url))
 
 	if _, err := c.makeRequest(logger, url, http.MethodDelete, nil, nil); err != nil { // nolint: bodyclose
 		logger.Error("failed to delete namespace", zap.Error(err))
@@ -462,7 +462,7 @@ func (c *CoordinatorClient) InitM3msgTopic(
 ) (admin.TopicGetResponse, error) {
 	url := c.makeURL(topic.InitURL)
 	logger := c.logger.With(
-		ZapMethod("initM3msgTopic"),
+		zapMethod("initM3msgTopic"),
 		zap.String("url", url),
 		zap.String("request", initRequest.String()),
 		zap.String("topic", fmt.Sprintf("%v", topicOpts)))
@@ -489,7 +489,7 @@ func (c *CoordinatorClient) GetM3msgTopic(
 ) (admin.TopicGetResponse, error) {
 	url := c.makeURL(topic.GetURL)
 	logger := c.logger.With(
-		ZapMethod("getM3msgTopic"), zap.String("url", url),
+		zapMethod("getM3msgTopic"), zap.String("url", url),
 		zap.String("topic", fmt.Sprintf("%v", topicOpts)))
 
 	resp, err := c.makeRequest(logger, url, topic.GetHTTPMethod, nil, m3msgTopicOptionsToMap(topicOpts))
@@ -516,7 +516,7 @@ func (c *CoordinatorClient) AddM3msgTopicConsumer(
 ) (admin.TopicGetResponse, error) {
 	url := c.makeURL(topic.AddURL)
 	logger := c.logger.With(
-		ZapMethod("addM3msgTopicConsumer"),
+		zapMethod("addM3msgTopicConsumer"),
 		zap.String("url", url),
 		zap.String("request", addRequest.String()),
 		zap.String("topic", fmt.Sprintf("%v", topicOpts)))
@@ -557,7 +557,7 @@ func (c *CoordinatorClient) WriteCarbon(
 	url string, metric string, v float64, t time.Time,
 ) error {
 	logger := c.logger.With(
-		ZapMethod("writeCarbon"), zap.String("url", url),
+		zapMethod("writeCarbon"), zap.String("url", url),
 		zap.String("at time", time.Now().String()),
 		zap.String("at ts", t.String()))
 
@@ -623,7 +623,7 @@ func (c *CoordinatorClient) WritePromWithRequest(writeRequest prompb.WriteReques
 	url := c.makeURL("api/v1/prom/remote/write")
 
 	logger := c.logger.With(
-		ZapMethod("writeProm"), zap.String("url", url),
+		zapMethod("writeProm"), zap.String("url", url),
 		zap.String("request", writeRequest.String()))
 
 	body, err := proto.Marshal(&writeRequest)
@@ -697,7 +697,7 @@ func (c *CoordinatorClient) ApplyKVUpdate(update string) error {
 	url := c.makeURL("api/v1/kvstore")
 
 	logger := c.logger.With(
-		ZapMethod("ApplyKVUpdate"), zap.String("url", url),
+		zapMethod("ApplyKVUpdate"), zap.String("url", url),
 		zap.String("update", update))
 
 	data := bytes.NewBuffer([]byte(update))
@@ -731,7 +731,7 @@ func (c *CoordinatorClient) query(
 ) error {
 	url := c.makeURL(query)
 	logger := c.logger.With(
-		ZapMethod("query"), zap.String("url", url), zap.Any("headers", headers))
+		zapMethod("query"), zap.String("url", url), zap.Any("headers", headers))
 	logger.Info("running")
 	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
 	if err != nil {
@@ -962,7 +962,7 @@ func (c *CoordinatorClient) runQuery(
 ) (string, error) {
 	url := c.makeURL(query)
 	logger := c.logger.With(
-		ZapMethod("query"), zap.String("url", url), zap.Any("headers", headers))
+		zapMethod("query"), zap.String("url", url), zap.Any("headers", headers))
 	logger.Info("running")
 	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
 	if err != nil {
@@ -1000,7 +1000,7 @@ func (c *CoordinatorClient) runQuery(
 func (c *CoordinatorClient) RunQuery(
 	verifier ResponseVerifier, query string, headers map[string][]string,
 ) error {
-	logger := c.logger.With(ZapMethod("runQuery"),
+	logger := c.logger.With(zapMethod("runQuery"),
 		zap.String("query", query))
 	err := c.retryFunc(func() error {
 		err := c.query(verifier, query, headers)
@@ -1067,7 +1067,7 @@ func (c *CoordinatorClient) GraphiteQuery(
 
 	url := c.makeURL(queryStr)
 	logger := c.logger.With(
-		ZapMethod("graphiteQuery"), zap.String("url", url))
+		zapMethod("graphiteQuery"), zap.String("url", url))
 	logger.Info("running")
 	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
 	if err != nil {
diff --git a/src/integration/resources/docker/dockerexternal/etcd.go b/src/integration/resources/docker/dockerexternal/etcd.go
index a9b3ed0023..bd67c58f05 100644
--- a/src/integration/resources/docker/dockerexternal/etcd.go
+++ b/src/integration/resources/docker/dockerexternal/etcd.go
@@ -175,8 +175,6 @@ func (c *EtcdNode) Setup(ctx context.Context) (closeErr error) {
 	// This is coming from the equivalent of docker inspect <container_id>
 	portBinds := container.NetworkSettings.Ports["2379/tcp"]
 
-	// If running in a docker container e.g. on buildkite, route to etcd using the published port on the *host* machine.
-	// See also http://github.com/m3db/m3/blob/master/docker-compose.yml#L16-L16
 	ipAddr := "127.0.0.1"
 	_, err = net.ResolveIPAddr("ip4", "host.docker.internal")
 	if err == nil {
diff --git a/src/integration/resources/inprocess/aggregator.go b/src/integration/resources/inprocess/aggregator.go
index 0192d7822f..fc76aef7c0 100644
--- a/src/integration/resources/inprocess/aggregator.go
+++ b/src/integration/resources/inprocess/aggregator.go
@@ -33,6 +33,7 @@ import (
 	m3agg "github.com/m3db/m3/src/aggregator/aggregator"
 	"github.com/m3db/m3/src/aggregator/server"
 	"github.com/m3db/m3/src/aggregator/tools/deploy"
+	etcdclient "github.com/m3db/m3/src/cluster/client/etcd"
 	"github.com/m3db/m3/src/cmd/services/m3aggregator/config"
 	"github.com/m3db/m3/src/integration/resources"
 	nettest "github.com/m3db/m3/src/integration/resources/net"
@@ -63,12 +64,16 @@ type Aggregator struct {
 
 // AggregatorOptions are options of starting an in-process aggregator.
 type AggregatorOptions struct {
+	// EtcdEndpoints are the endpoints this aggregator should use to connect to etcd.
+	EtcdEndpoints []string
+
 	// Logger is the logger to use for the in-process aggregator.
 	Logger *zap.Logger
 	// StartFn is a custom function that can be used to start the Aggregator.
 	StartFn AggregatorStartFn
 	// Start indicates whether to start the aggregator instance
 	Start bool
+
 	// GeneratePorts will automatically update the config to use open ports
 	// if set to true. If false, configuration is used as-is re: ports.
 	GeneratePorts bool
@@ -286,6 +291,10 @@ func updateAggregatorConfig(
 		}
 	}
 
+	kvCfg := cfg.KVClientOrDefault()
+	cfg.KVClient = &kvCfg
+	updateEtcdEndpoints(opts.EtcdEndpoints, cfg.KVClient.Etcd)
+
 	// Replace any filepath with a temporary directory
 	cfg, tmpDirs, err = updateAggregatorFilepaths(cfg)
 	if err != nil {
@@ -295,6 +304,11 @@ func updateAggregatorConfig(
 	return cfg, tmpDirs, nil
 }
 
+func updateEtcdEndpoints(etcdEndpoints []string, etcdCfg *etcdclient.Configuration) {
+	etcdCfg.ETCDClusters[0].Endpoints = etcdEndpoints
+	etcdCfg.ETCDClusters[0].AutoSyncInterval = -1
+}
+
 func updateAggregatorHostID(cfg config.Configuration) config.Configuration {
 	hostID := uuid.New().String()
 	aggCfg := cfg.AggregatorOrDefault()
diff --git a/src/integration/resources/inprocess/cluster.go b/src/integration/resources/inprocess/cluster.go
index 2b2dbc2303..01630b2b81 100644
--- a/src/integration/resources/inprocess/cluster.go
+++ b/src/integration/resources/inprocess/cluster.go
@@ -21,11 +21,12 @@
 package inprocess
 
 import (
+	"context"
 	"errors"
 	"fmt"
-	"net"
-	"strconv"
+	"time"
 
+	etcdclient "github.com/m3db/m3/src/cluster/client/etcd"
 	aggcfg "github.com/m3db/m3/src/cmd/services/m3aggregator/config"
 	dbcfg "github.com/m3db/m3/src/cmd/services/m3dbnode/config"
 	coordinatorcfg "github.com/m3db/m3/src/cmd/services/m3query/config"
@@ -34,13 +35,15 @@ import (
 	"github.com/m3db/m3/src/dbnode/environment"
 	"github.com/m3db/m3/src/dbnode/persist/fs"
 	"github.com/m3db/m3/src/integration/resources"
-	nettest "github.com/m3db/m3/src/integration/resources/net"
+	"github.com/m3db/m3/src/integration/resources/docker/dockerexternal"
 	"github.com/m3db/m3/src/query/storage/m3"
 	xconfig "github.com/m3db/m3/src/x/config"
 	"github.com/m3db/m3/src/x/config/hostid"
 	xerrors "github.com/m3db/m3/src/x/errors"
+	"github.com/m3db/m3/src/x/instrument"
 
 	"github.com/google/uuid"
+	"github.com/ory/dockertest/v3"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
@@ -122,17 +125,51 @@ func NewClusterConfigsFromConfigFile(
 // NewClusterConfigsFromYAML creates a new ClusterConfigs object from YAML strings
 // representing component configs.
 func NewClusterConfigsFromYAML(dbnodeYaml string, coordYaml string, aggYaml string) (ClusterConfigs, error) {
-	var dbCfg dbcfg.Configuration
+	// "db":
+	//  discovery:
+	//    "config":
+	//      "service":
+	//        "etcdClusters":
+	//          - "endpoints": ["http://127.0.0.1:2379"]
+	//            "zone": "embedded"
+	//        "service": "m3db"
+	//        "zone": "embedded"
+	//        "env": "default_env"
+	etcdClientCfg := &etcdclient.Configuration{
+		Zone:    "embedded",
+		Env:     "default_env",
+		Service: "m3db",
+		ETCDClusters: []etcdclient.ClusterConfig{{
+			Zone:      "embedded",
+			Endpoints: []string{"http://127.0.0.1:2379"},
+		}},
+	}
+	var dbCfg = dbcfg.Configuration{
+		DB: &dbcfg.DBConfiguration{
+			Discovery: &discovery.Configuration{
+				Config: &environment.Configuration{
+					Services: environment.DynamicConfiguration{{
+						Service: etcdClientCfg,
+					}},
+				},
+			},
+		},
+	}
 	if err := yaml.Unmarshal([]byte(dbnodeYaml), &dbCfg); err != nil {
 		return ClusterConfigs{}, err
 	}
 
-	var coordCfg coordinatorcfg.Configuration
+	var coordCfg = coordinatorcfg.Configuration{
+		ClusterManagement: coordinatorcfg.ClusterManagementConfiguration{
+			Etcd: etcdClientCfg,
+		},
+	}
 	if err := yaml.Unmarshal([]byte(coordYaml), &coordCfg); err != nil {
 		return ClusterConfigs{}, err
 	}
 
-	var aggCfg aggcfg.Configuration
+	var aggCfg = aggcfg.Configuration{}
+
 	if aggYaml != "" {
 		if err := yaml.Unmarshal([]byte(aggYaml), &aggCfg); err != nil {
 			return ClusterConfigs{}, err
@@ -164,7 +201,7 @@ func NewCluster(
 func NewClusterFromSpecification(
 	specs ClusterSpecification,
 	opts resources.ClusterOptions,
-) (resources.M3Resources, error) {
+) (_ resources.M3Resources, finalErr error) {
 	if err := opts.Validate(); err != nil {
 		return nil, err
 	}
@@ -175,6 +212,7 @@ func NewClusterFromSpecification(
 	}
 
 	var (
+		etcd  *dockerexternal.EtcdNode
 		coord resources.Coordinator
 		nodes = make(resources.Nodes, 0, len(specs.Configs.DBNodes))
 		aggs  = make(resources.Aggregators, 0, len(specs.Configs.Aggregators))
@@ -185,13 +223,38 @@ func NewClusterFromSpecification(
 	// Ensure that once we start creating resources, they all get cleaned up even if the function
 	// fails half way.
 	defer func() {
-		if err != nil {
-			cleanup(logger, nodes, coord, aggs)
+		if finalErr != nil {
+			cleanup(logger, etcd, nodes, coord, aggs)
 		}
 	}()
 
+	etcdEndpoints := opts.EtcdEndpoints
+	if len(opts.EtcdEndpoints) == 0 {
+		// TODO: amainsd: maybe not the cleanest place to do this.
+		pool, err := dockertest.NewPool("")
+		if err != nil {
+			return nil, err
+		}
+		etcd, err = dockerexternal.NewEtcd(pool, instrument.NewOptions())
+		if err != nil {
+			return nil, err
+		}
+
+		// TODO(amains): etcd *needs* to be setup before the coordinator, because ConfigurePlacementsForAggregation spins
+		// up a dedicated coordinator for some reason. Either clean this up or just accept it.
+		if err := etcd.Setup(context.TODO()); err != nil {
+			return nil, err
+		}
+		etcdEndpoints = []string{fmt.Sprintf(etcd.Address())}
+	}
+
+	updateEtcdEndpoints := func(etcdCfg *etcdclient.Configuration) {
+		etcdCfg.ETCDClusters[0].Endpoints = etcdEndpoints
+		etcdCfg.ETCDClusters[0].AutoSyncInterval = -1
+	}
 	for i := 0; i < len(specs.Configs.DBNodes); i++ {
 		var node resources.Node
+		updateEtcdEndpoints(specs.Configs.DBNodes[i].DB.Discovery.Config.Services[0].Service)
 		node, err = NewDBNode(specs.Configs.DBNodes[i], specs.Options.DBNode[i])
 		if err != nil {
 			return nil, err
@@ -204,6 +267,7 @@ func NewClusterFromSpecification(
 		agg, err = NewAggregator(aggCfg, AggregatorOptions{
 			GeneratePorts:  true,
 			GenerateHostID: false,
+			EtcdEndpoints:  etcdEndpoints,
 		})
 		if err != nil {
 			return nil, err
@@ -211,6 +275,7 @@ func NewClusterFromSpecification(
 		aggs = append(aggs, agg)
 	}
 
+	updateEtcdEndpoints(specs.Configs.Coordinator.ClusterManagement.Etcd)
 	coord, err = NewCoordinator(
 		specs.Configs.Coordinator,
 		CoordinatorOptions{GeneratePorts: opts.Coordinator.GeneratePorts},
@@ -220,7 +285,7 @@ func NewClusterFromSpecification(
 	}
 
 	if err = ConfigurePlacementsForAggregation(nodes, coord, aggs, specs, opts); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to setup placements for aggregation: %w", err)
 	}
 
 	// Start all the configured resources.
@@ -228,6 +293,7 @@ func NewClusterFromSpecification(
 		Coordinator: coord,
 		DBNodes:     nodes,
 		Aggregators: aggs,
+		Etcd:        etcd,
 	})
 	m3.Start()
 
@@ -371,13 +437,13 @@ func GenerateDBNodeConfigsForCluster(
 	// the etcd server (i.e. seed node).
 	hostID := uuid.NewString()
 	defaultDBNodesCfg := configs.DBNode
-	discoveryCfg, envConfig, err := generateDefaultDiscoveryConfig(
-		defaultDBNodesCfg,
-		hostID,
-		generatePortsAndIDs)
-	if err != nil {
-		return nil, nil, environment.Configuration{}, err
+
+	if configs.DBNode.DB.Discovery == nil {
+		return nil, nil, environment.Configuration{}, errors.New(
+			"configuration must specify at least `discovery`" +
+				" in order to construct an etcd client")
 	}
+	discoveryCfg, envConfig := configs.DBNode.DB.Discovery, configs.DBNode.DB.Discovery.Config
 
 	var (
 		defaultDBNodeOpts = DBNodeOptions{
@@ -389,8 +455,7 @@ func GenerateDBNodeConfigsForCluster(
 		nodeOpts = make([]DBNodeOptions, 0, numNodes)
 	)
 	for i := 0; i < int(numNodes); i++ {
-		var cfg dbcfg.Configuration
-		cfg, err = defaultDBNodesCfg.DeepCopy()
+		cfg, err := defaultDBNodesCfg.DeepCopy()
 		if err != nil {
 			return nil, nil, environment.Configuration{}, err
 		}
@@ -404,68 +469,31 @@ func GenerateDBNodeConfigsForCluster(
 				Value:    &hostID,
 			}
 		}
-		cfg.DB.Discovery = &discoveryCfg
+		cfg.DB.Discovery = discoveryCfg
 
 		cfgs = append(cfgs, cfg)
 		nodeOpts = append(nodeOpts, dbnodeOpts)
 	}
 
-	return cfgs, nodeOpts, envConfig, nil
+	return cfgs, nodeOpts, *envConfig, nil
 }
 
-// generateDefaultDiscoveryConfig handles creating the correct config
-// for having an embedded ETCD server with the correct server and
-// client configuration.
-func generateDefaultDiscoveryConfig(
-	cfg dbcfg.Configuration,
-	hostID string,
-	generateETCDPorts bool,
-) (discovery.Configuration, environment.Configuration, error) {
-	discoveryConfig := cfg.DB.DiscoveryOrDefault()
-	envConfig, err := discoveryConfig.EnvironmentConfig(hostID)
-	if err != nil {
-		return discovery.Configuration{}, environment.Configuration{}, err
-	}
-
-	var (
-		etcdClientPort = dbcfg.DefaultEtcdClientPort
-		etcdServerPort = dbcfg.DefaultEtcdServerPort
-	)
-	if generateETCDPorts {
-		etcdClientPort, err = nettest.GetAvailablePort()
-		if err != nil {
-			return discovery.Configuration{}, environment.Configuration{}, err
-		}
-
-		etcdServerPort, err = nettest.GetAvailablePort()
-		if err != nil {
-			return discovery.Configuration{}, environment.Configuration{}, err
-		}
-	}
+func cleanup(
+	logger *zap.Logger,
+	etcd *dockerexternal.EtcdNode,
+	nodes resources.Nodes,
+	coord resources.Coordinator,
+	aggs resources.Aggregators,
+) {
+	var multiErr xerrors.MultiError
 
-	etcdServerURL := fmt.Sprintf("http://0.0.0.0:%d", etcdServerPort)
-	etcdClientAddr := net.JoinHostPort("0.0.0.0", strconv.Itoa(etcdClientPort))
-	etcdClientURL := fmt.Sprintf("http://0.0.0.0:%d", etcdClientPort)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
 
-	envConfig.SeedNodes.InitialCluster[0].Endpoint = etcdServerURL
-	envConfig.SeedNodes.InitialCluster[0].HostID = hostID
-	envConfig.Services[0].Service.ETCDClusters[0].Endpoints = []string{etcdClientAddr}
-	if generateETCDPorts {
-		envConfig.SeedNodes.ListenPeerUrls = []string{etcdServerURL}
-		envConfig.SeedNodes.ListenClientUrls = []string{etcdClientURL}
-		envConfig.SeedNodes.InitialAdvertisePeerUrls = []string{etcdServerURL}
-		envConfig.SeedNodes.AdvertiseClientUrls = []string{etcdClientURL}
+	if etcd != nil {
+		multiErr = multiErr.Add(etcd.Close(ctx))
 	}
 
-	configType := discovery.ConfigType
-	return discovery.Configuration{
-		Type:   &configType,
-		Config: &envConfig,
-	}, envConfig, nil
-}
-
-func cleanup(logger *zap.Logger, nodes resources.Nodes, coord resources.Coordinator, aggs resources.Aggregators) {
-	var multiErr xerrors.MultiError
 	for _, n := range nodes {
 		multiErr = multiErr.Add(n.Close())
 	}
diff --git a/src/integration/resources/inprocess/coordinator_test.go b/src/integration/resources/inprocess/coordinator_test.go
index e818926439..9f9e776139 100644
--- a/src/integration/resources/inprocess/coordinator_test.go
+++ b/src/integration/resources/inprocess/coordinator_test.go
@@ -1,4 +1,6 @@
+//go:build test_harness
 // +build test_harness
+
 // Copyright (c) 2021  Uber Technologies, Inc.
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -22,17 +24,21 @@
 package inprocess
 
 import (
+	"context"
 	"testing"
 
 	"github.com/m3db/m3/src/cluster/generated/proto/placementpb"
 	"github.com/m3db/m3/src/cluster/placement"
 	"github.com/m3db/m3/src/integration/resources"
+	"github.com/m3db/m3/src/integration/resources/docker/dockerexternal"
 	"github.com/m3db/m3/src/msg/generated/proto/topicpb"
 	"github.com/m3db/m3/src/msg/topic"
 	"github.com/m3db/m3/src/query/generated/proto/admin"
 	"github.com/m3db/m3/src/query/generated/proto/prompb"
 	"github.com/m3db/m3/src/query/storage"
+	"github.com/m3db/m3/src/x/instrument"
 	xtime "github.com/m3db/m3/src/x/time"
+	"github.com/ory/dockertest/v3"
 
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
@@ -40,6 +46,16 @@ import (
 )
 
 func TestNewCoordinator(t *testing.T) {
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err)
+
+	etcd, err := dockerexternal.NewEtcd(pool, instrument.NewOptions(), dockerexternal.EtcdClusterPort(2379))
+	require.NoError(t, err)
+	require.NoError(t, etcd.Setup(context.TODO()))
+	t.Cleanup(func() {
+		require.NoError(t, etcd.Close(context.TODO()))
+	})
+
 	dbnode, err := NewDBNodeFromYAML(defaultDBNodeConfig, DBNodeOptions{Start: true})
 	require.NoError(t, err)
 	defer func() {
diff --git a/src/integration/resources/inprocess/dbnode_test.go b/src/integration/resources/inprocess/dbnode_test.go
index 4528a7b58a..f08fa81c00 100644
--- a/src/integration/resources/inprocess/dbnode_test.go
+++ b/src/integration/resources/inprocess/dbnode_test.go
@@ -1,4 +1,6 @@
+//go:build test_harness
 // +build test_harness
+
 // Copyright (c) 2021  Uber Technologies, Inc.
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
diff --git a/src/integration/resources/inprocess/inprocess.go b/src/integration/resources/inprocess/inprocess.go
index a7a5af95f2..b0e9711bf0 100644
--- a/src/integration/resources/inprocess/inprocess.go
+++ b/src/integration/resources/inprocess/inprocess.go
@@ -21,7 +21,10 @@
 package inprocess
 
 import (
+	"context"
+
 	"github.com/m3db/m3/src/integration/resources"
+	"github.com/m3db/m3/src/integration/resources/docker/dockerexternal"
 	"github.com/m3db/m3/src/x/errors"
 )
 
@@ -29,11 +32,13 @@ type inprocessM3Resources struct {
 	coordinator resources.Coordinator
 	dbNodes     resources.Nodes
 	aggregators resources.Aggregators
+	etcd        *dockerexternal.EtcdNode
 }
 
 // ResourceOptions are the options for creating new
 // resources.M3Resources.
 type ResourceOptions struct {
+	Etcd        *dockerexternal.EtcdNode
 	Coordinator resources.Coordinator
 	DBNodes     resources.Nodes
 	Aggregators resources.Aggregators
@@ -43,6 +48,7 @@ type ResourceOptions struct {
 // backed by in-process implementations of the M3 components.
 func NewM3Resources(options ResourceOptions) resources.M3Resources {
 	return &inprocessM3Resources{
+		etcd:        options.Etcd,
 		coordinator: options.Coordinator,
 		dbNodes:     options.DBNodes,
 		aggregators: options.Aggregators,
@@ -73,6 +79,11 @@ func (i *inprocessM3Resources) Cleanup() error {
 		err = err.Add(a.Close())
 	}
 
+	if i.etcd != nil {
+		ctx := context.TODO()
+		err = err.Add(i.etcd.Close(ctx))
+	}
+
 	return err.FinalError()
 }
 
diff --git a/src/integration/resources/options.go b/src/integration/resources/options.go
index d980c154f7..88eeddfe95 100644
--- a/src/integration/resources/options.go
+++ b/src/integration/resources/options.go
@@ -1,3 +1,23 @@
+// Copyright (c) 2022 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 package resources
 
 import (
@@ -8,6 +28,9 @@ import (
 // ClusterOptions contains options for spinning up a new M3 cluster
 // composed of in-process components.
 type ClusterOptions struct {
+	// EtcdEndpoints if provided, will be used directly instead of spinning up a dedicated etcd node for the cluster.
+	// By default, NewClusterFromSpecification will spin up and manage an etcd node itself.
+	EtcdEndpoints []string
 	// DBNode contains cluster options for spinning up dbnodes.
 	DBNode *DBNodeClusterOptions
 	// Aggregator is the optional cluster options for spinning up aggregators.

From abd72e3d89f90e4afe2469a026e7abff054779ab Mon Sep 17 00:00:00 2001
From: Andrew Mains <amains@uber.com>
Date: Fri, 2 Sep 2022 12:35:16 -0400
Subject: [PATCH 2/2] etcd_docker 5: Incorporate docker based etcd approach
 into docker integration tests.

PR 5 for https://github.com/m3db/m3/issues/4144

This PR makes the docker integration tests use containerized etcd.
Previously, these relied on M3DB running an embbeded etcd server.

There's no inherent need for this, and it opens us up to dependency issues as described
in the linked github issue.

Note: there are a handful that require multiple servers; these are currently "skipped" (commented). I intend to bring those
back at a later date..

commit-id:e67a5172
---
 .../aggregator/docker-compose.yml             | 23 ++++-
 .../aggregator/m3aggregator.yml               |  2 +-
 .../aggregator/m3coordinator.yml              |  2 +-
 .../aggregator/test.sh                        |  2 +
 .../aggregator_legacy/docker-compose.yml      | 19 +++-
 .../aggregator_legacy/m3aggregator.yml        |  2 +-
 .../aggregator_legacy/m3coordinator.yml       |  2 +-
 .../aggregator_legacy/test.sh                 |  3 +
 .../carbon/docker-compose.yml                 | 19 +++-
 .../carbon/m3coordinator.yml                  |  2 +-
 .../docker-integration-tests/carbon/test.sh   |  5 +-
 .../cold_writes_simple/docker-compose.yml     | 19 +++-
 .../cold_writes_simple/m3coordinator.yml      |  2 +-
 .../cold_writes_simple/test.sh                |  5 +-
 .../docker-compose.yml                        | 19 +++-
 .../m3coordinator.yml                         |  2 +-
 .../coordinator_config_rules/test.sh          |  3 +-
 .../coordinator_noop/docker-compose.yml       | 43 +++-------
 .../coordinator_noop/m3coordinator.yml        |  2 +-
 .../coordinator_noop/test.sh                  |  3 +-
 .../docker-compose.yml                        | 43 +++-------
 .../docker-compose-etcd.yml                   | 15 ++++
 .../m3coordinator.Dockerfile                  |  2 +-
 .../m3dbnode.Dockerfile                       |  4 +-
 .../docker-compose.yml                        | 53 ++++++------
 .../m3aggregator.yml                          |  2 +-
 .../m3coordinator-admin.yml                   |  2 +-
 .../m3coordinator.yml                         |  2 +-
 .../prom_remote_write_backend/test.sh         |  2 +-
 .../prometheus/docker-compose.yml             | 21 ++++-
 .../prometheus/m3coordinator.yml              |  2 +-
 .../repair/docker-compose.yml                 | 21 ++++-
 .../repair/m3coordinator.yml                  |  2 +-
 .../repair/m3dbnode.yml                       |  7 +-
 .../docker-integration-tests/repair/test.sh   |  4 +-
 scripts/docker-integration-tests/run.sh       | 86 +++++++++++--------
 scripts/docker-integration-tests/setup.sh     |  8 +-
 .../simple_v2_batch_apis/docker-compose.yml   | 19 +++-
 .../simple_v2_batch_apis/m3coordinator.yml    |  2 +-
 .../config/m3dbnode-local-docker-etcd.yml     | 11 +++
 .../m3coordinator-local-docker-etcd.yml       | 16 ++++
 41 files changed, 314 insertions(+), 189 deletions(-)
 create mode 100644 scripts/docker-integration-tests/docker-compose-etcd.yml
 create mode 100644 src/dbnode/config/m3dbnode-local-docker-etcd.yml
 create mode 100644 src/query/config/m3coordinator-local-docker-etcd.yml

diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml
index c93b41ee25..9ad000907d 100644
--- a/scripts/docker-integration-tests/aggregator/docker-compose.yml
+++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml
@@ -1,17 +1,28 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
       - "7201"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
       - "0.0.0.0:7201:7201"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -26,6 +37,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   m3aggregator01:
     expose:
       - "6001"
@@ -38,6 +51,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     networks:
       - backend
@@ -46,5 +61,7 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml
index c2f26330bd..1aa1c7c091 100644
--- a/scripts/docker-integration-tests/aggregator/m3aggregator.yml
+++ b/scripts/docker-integration-tests/aggregator/m3aggregator.yml
@@ -38,7 +38,7 @@ kvClient:
         autoSyncInterval: 10m
         dialTimeout: 1m
         endpoints:
-          - dbnode01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:
diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
index dfc757b8c4..a1e44edda4 100644
--- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml
+++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
@@ -24,7 +24,7 @@ clusters:
               autoSyncInterval: 10m
               dialTimeout: 1m
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   rules:
diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh
index a65b912c8a..a5b44122c8 100755
--- a/scripts/docker-integration-tests/aggregator/test.sh
+++ b/scripts/docker-integration-tests/aggregator/test.sh
@@ -14,6 +14,8 @@ echo "Pull containers required for test"
 docker pull $PROMREMOTECLI_IMAGE
 docker pull $JQ_IMAGE
 
+docker-compose -f ${COMPOSE_FILE} up -d etcd
+
 echo "Run m3dbnode"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 
diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
index c93b41ee25..76eaac7862 100644
--- a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
+++ b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
@@ -1,17 +1,24 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
       - "7201"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
       - "0.0.0.0:7201:7201"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -26,6 +33,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   m3aggregator01:
     expose:
       - "6001"
@@ -38,6 +47,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     networks:
       - backend
@@ -46,5 +57,7 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml
index 569ea9a9c2..e10c7cbe96 100644
--- a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml
+++ b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml
@@ -57,7 +57,7 @@ kvClient:
     etcdClusters:
       - zone: embedded
         endpoints:
-          - dbnode01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:
diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
index 281662faca..d15fd1a78c 100644
--- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
+++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
@@ -22,7 +22,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   remoteAggregator:
diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh
index e101eae186..7f5050005b 100755
--- a/scripts/docker-integration-tests/aggregator_legacy/test.sh
+++ b/scripts/docker-integration-tests/aggregator_legacy/test.sh
@@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD)
 COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
 export REVISION
 
+echo "Run etcd"
+docker-compose -f ${COMPOSE_FILE} up -d etcd
+
 echo "Run m3dbnode"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 
diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml
index 53a28f0b88..a40ac2024a 100644
--- a/scripts/docker-integration-tests/carbon/docker-compose.yml
+++ b/scripts/docker-integration-tests/carbon/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +35,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
index 7f33d8701c..b069fbf7b6 100644
--- a/scripts/docker-integration-tests/carbon/m3coordinator.yml
+++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -9,7 +9,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 carbon:
   findResultsIncludeBothExpandableAndLeaf: true
diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh
index f1c499a71e..6c811cb3b8 100755
--- a/scripts/docker-integration-tests/carbon/test.sh
+++ b/scripts/docker-integration-tests/carbon/test.sh
@@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected
 export REVISION
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d
 
 # Think of this as a defer func() in golang
 METRIC_EMIT_PID="-1"
@@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum
 
 # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s)
 end=$(date +%s)
-start=$(($end-20)) 
+start=$(($end-20))
 # 1. no max datapoints set, should not adjust number of datapoints coming back
 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end"
 # 2. max datapoints with LTTB, should be an existing value (i.e. 42)
diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml
index 53a28f0b88..a40ac2024a 100644
--- a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml
+++ b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +35,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
index cc33cf4021..dc075a01fe 100644
--- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
+++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
@@ -13,4 +13,4 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh
index 999ef1b20d..235c870e2b 100755
--- a/scripts/docker-integration-tests/cold_writes_simple/test.sh
+++ b/scripts/docker-integration-tests/cold_writes_simple/test.sh
@@ -8,9 +8,8 @@ SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/cold_writes_simple
 COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml
 export REVISION
 
-echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01
+echo "Run etcd, m3dbnode and m3coordinator containers"
+docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes
 
 # Think of this as a defer func() in golang
 function defer {
diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml
index 53a28f0b88..a40ac2024a 100644
--- a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml
+++ b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +35,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
index 952c9a7ddf..3ca941aa8d 100644
--- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
+++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
@@ -9,7 +9,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   rules:
diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh
index a1590983c8..48dce08870 100755
--- a/scripts/docker-integration-tests/coordinator_config_rules/test.sh
+++ b/scripts/docker-integration-tests/coordinator_config_rules/test.sh
@@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE
 docker pull $JQ_IMAGE
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d
 
 # Think of this as a defer func() in golang
 function defer {
diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml
index ee8207bd26..edb2a7528c 100644
--- a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml
+++ b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml
@@ -1,5 +1,16 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   coordinator01:
     expose:
       - "7201"
@@ -10,33 +21,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
-  etcd01:
-    expose:
-      - "2379-2380"
-    ports:
-      - "0.0.0.0:2379-2380:2379-2380"
-    networks:
-      - backend
-    image: quay.io/coreos/etcd:v3.4.3
-    command:
-      - "etcd"
-      - "--name"
-      - "etcd01"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2380"
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--advertise-client-urls"
-      - "http://etcd01:2379"
-      - "--initial-cluster-token"
-      - "etcd-cluster-1"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd01:2380"
-      - "--initial-cluster"
-      - "etcd01=http://etcd01:2380"
-      - "--initial-cluster-state"
-      - "new"
-      - "--data-dir"
-      - "/var/lib/etcd"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml
index c3c08c0104..6a261ecf26 100644
--- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml
+++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml
@@ -23,7 +23,7 @@ clusterManagement:
     etcdClusters:
     - zone: embedded
       endpoints:
-      - etcd01:2379
+      - etcd:2379
 
 tagOptions:
   idScheme: quoted
diff --git a/scripts/docker-integration-tests/coordinator_noop/test.sh b/scripts/docker-integration-tests/coordinator_noop/test.sh
index 738ea863d0..0f91ec91da 100755
--- a/scripts/docker-integration-tests/coordinator_noop/test.sh
+++ b/scripts/docker-integration-tests/coordinator_noop/test.sh
@@ -9,8 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml
 export REVISION
 
 echo "Run coordinator with no etcd"
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd01
+docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes
 
 function defer {
   docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes
diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml
index e155eb5334..c39ef36c0c 100644
--- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml
+++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml
@@ -1,5 +1,16 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
@@ -14,33 +25,7 @@ services:
       - M3DB_HOST_ID=dbnode01
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
-  etcd01:
-    expose:
-      - "2379-2380"
-    ports:
-      - "0.0.0.0:2379-2380:2379-2380"
-    networks:
-      - backend
-    image: quay.io/coreos/etcd:v3.4.3
-    command:
-      - "etcd"
-      - "--name"
-      - "etcd01"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2380"
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--advertise-client-urls"
-      - "http://etcd01:2379"
-      - "--initial-cluster-token"
-      - "etcd-cluster-1"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd01:2380"
-      - "--initial-cluster"
-      - "etcd01=http://etcd01:2380"
-      - "--initial-cluster-state"
-      - "new"
-      - "--data-dir"
-      - "/var/lib/etcd"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/docker-compose-etcd.yml b/scripts/docker-integration-tests/docker-compose-etcd.yml
new file mode 100644
index 0000000000..90af19b550
--- /dev/null
+++ b/scripts/docker-integration-tests/docker-compose-etcd.yml
@@ -0,0 +1,15 @@
+version: "3.5"
+services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+    networks:
+      - backend
+
+networks:
+  backend:
diff --git a/scripts/docker-integration-tests/m3coordinator.Dockerfile b/scripts/docker-integration-tests/m3coordinator.Dockerfile
index 0319613977..489030dbea 100644
--- a/scripts/docker-integration-tests/m3coordinator.Dockerfile
+++ b/scripts/docker-integration-tests/m3coordinator.Dockerfile
@@ -4,7 +4,7 @@ LABEL maintainer="The M3DB Authors <m3db@googlegroups.com>"
 RUN mkdir -p /bin
 RUN mkdir -p /etc/m3coordinator
 ADD ./m3coordinator /bin/
-ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml
+ADD ./m3coordinator-local-docker-etcd.yml /etc/m3coordinator/m3coordinator.yml
 
 EXPOSE 7201/tcp 7203/tcp
 
diff --git a/scripts/docker-integration-tests/m3dbnode.Dockerfile b/scripts/docker-integration-tests/m3dbnode.Dockerfile
index a352ad4bf5..6d430fe803 100644
--- a/scripts/docker-integration-tests/m3dbnode.Dockerfile
+++ b/scripts/docker-integration-tests/m3dbnode.Dockerfile
@@ -4,9 +4,9 @@ LABEL maintainer="The M3DB Authors <m3db@googlegroups.com>"
 RUN mkdir -p /bin
 RUN mkdir -p /etc/m3dbnode
 ADD ./m3dbnode /bin/
-ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml
+ADD ./m3dbnode-local-docker-etcd.yml /etc/m3dbnode/m3dbnode.yml
 
-EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp
+EXPOSE 7201/tcp 7203/tcp 9000-9004/tcp
 
 ENV PANIC_ON_INVARIANT_VIOLATED=true
 
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml
index 6c38ae39de..f89e401db3 100644
--- a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml
+++ b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml
@@ -1,5 +1,16 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   m3aggregator01:
     expose:
       - "6001"
@@ -12,6 +23,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     expose:
       - "6002"
@@ -24,6 +37,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -34,6 +49,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
   coordinatoradmin:
     expose:
       - "7201"
@@ -44,6 +61,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   prometheusraw:
     expose:
       - "9090"
@@ -60,6 +79,8 @@ services:
       - "--web.console.libraries=/usr/share/prometheus/console_libraries"
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--enable-feature=remote-write-receiver"
+    depends_on:
+      - etcd
   prometheusagg:
     expose:
       - "9091"
@@ -76,33 +97,7 @@ services:
       - "--web.console.libraries=/usr/share/prometheus/console_libraries"
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--enable-feature=remote-write-receiver"
-  etcd01:
-    expose:
-      - "2379-2380"
-    ports:
-      - "0.0.0.0:2379-2380:2379-2380"
-    networks:
-      - backend
-    image: quay.io/coreos/etcd:v3.4.3
-    command:
-      - "etcd"
-      - "--name"
-      - "etcd01"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2380"
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--advertise-client-urls"
-      - "http://etcd01:2379"
-      - "--initial-cluster-token"
-      - "etcd-cluster-1"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd01:2380"
-      - "--initial-cluster"
-      - "etcd01=http://etcd01:2380"
-      - "--initial-cluster-state"
-      - "new"
-      - "--data-dir"
-      - "/var/lib/etcd"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml
index 1d77b0a035..37ac3f2aa4 100644
--- a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml
+++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml
@@ -40,7 +40,7 @@ kvClient:
     etcdClusters:
       - zone: embedded
         endpoints:
-          - etcd01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml
index c3c08c0104..6a261ecf26 100644
--- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml
+++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml
@@ -23,7 +23,7 @@ clusterManagement:
     etcdClusters:
     - zone: embedded
       endpoints:
-      - etcd01:2379
+      - etcd:2379
 
 tagOptions:
   idScheme: quoted
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml
index d6c54c8430..935bc8ed5e 100644
--- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml
+++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml
@@ -36,7 +36,7 @@ clusterManagement:
     etcdClusters:
       - zone: embedded
         endpoints:
-          - etcd01:2379
+          - etcd:2379
 
 tagOptions:
   idScheme: quoted
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh
index 208bb88565..7ba4c0435e 100755
--- a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh
+++ b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh
@@ -20,7 +20,7 @@ docker pull $PROMREMOTECLI_IMAGE
 trap 'cleanup ${COMPOSE_FILE} ${TEST_SUCCESS}' EXIT
 
 echo "Run ETCD"
-docker-compose -f "${COMPOSE_FILE}" up -d etcd01
+docker-compose -f "${COMPOSE_FILE}" up -d etcd
 
 echo "Run Coordinator in Admin mode"
 docker-compose -f "${COMPOSE_FILE}" up -d coordinatoradmin
diff --git a/scripts/docker-integration-tests/prometheus/docker-compose.yml b/scripts/docker-integration-tests/prometheus/docker-compose.yml
index 03b839832b..2bfe100b1e 100644
--- a/scripts/docker-integration-tests/prometheus/docker-compose.yml
+++ b/scripts/docker-integration-tests/prometheus/docker-compose.yml
@@ -1,17 +1,28 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,6 +35,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
   prometheus01:
     expose:
       - "9090"
@@ -34,5 +47,7 @@ services:
     image: prom/prometheus:latest
     volumes:
       - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml
index 79f849a151..7ebe2792b3 100644
--- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml
+++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml
@@ -13,7 +13,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 query:
   restrictTags:
diff --git a/scripts/docker-integration-tests/repair/docker-compose.yml b/scripts/docker-integration-tests/repair/docker-compose.yml
index b91284adcf..bef03881d9 100644
--- a/scripts/docker-integration-tests/repair/docker-compose.yml
+++ b/scripts/docker-integration-tests/repair/docker-compose.yml
@@ -1,9 +1,19 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9012:9002"
       - "0.0.0.0:9013:9003"
@@ -14,10 +24,11 @@ services:
       - M3DB_HOST_ID=m3db_local_1
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
+    depends_on:
+      - etcd
   dbnode02:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9022:9002"
       - "0.0.0.0:9023:9003"
@@ -28,6 +39,8 @@ services:
       - M3DB_HOST_ID=m3db_local_2
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -42,5 +55,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml
index cc33cf4021..dc075a01fe 100644
--- a/scripts/docker-integration-tests/repair/m3coordinator.yml
+++ b/scripts/docker-integration-tests/repair/m3coordinator.yml
@@ -13,4 +13,4 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml
index 29dcc22d4b..62f3e88cf8 100644
--- a/scripts/docker-integration-tests/repair/m3dbnode.yml
+++ b/scripts/docker-integration-tests/repair/m3dbnode.yml
@@ -13,12 +13,7 @@ db:
         etcdClusters:
           - zone: embedded
             endpoints:
-              - dbnode01:2379
-      seedNodes:
-        initialCluster:
-          - hostID: m3db_local_1
-            endpoint: http://dbnode01:2380
-
+              - etcd:2379
   # Enable repairs.
   repair:
     enabled: true
diff --git a/scripts/docker-integration-tests/repair/test.sh b/scripts/docker-integration-tests/repair/test.sh
index ee32d632a9..4cb6e3b4f4 100755
--- a/scripts/docker-integration-tests/repair/test.sh
+++ b/scripts/docker-integration-tests/repair/test.sh
@@ -9,9 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml
 export REVISION
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode02
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes
 
 # Think of this as a defer func() in golang
 function defer {
diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh
index bead15b207..fa7f2901e3 100755
--- a/scripts/docker-integration-tests/run.sh
+++ b/scripts/docker-integration-tests/run.sh
@@ -3,45 +3,55 @@
 set -ex
 
 TESTS=(
-	scripts/docker-integration-tests/cold_writes_simple/test.sh
-	scripts/docker-integration-tests/prometheus_replication/test.sh
-	scripts/docker-integration-tests/carbon/test.sh
-	scripts/docker-integration-tests/aggregator/test.sh
-	scripts/docker-integration-tests/aggregator_legacy/test.sh
-	scripts/docker-integration-tests/query_fanout/test.sh
-	scripts/docker-integration-tests/repair/test.sh
-	scripts/docker-integration-tests/replication/test.sh
-	scripts/docker-integration-tests/multi_cluster_write/test.sh
-	scripts/docker-integration-tests/coordinator_config_rules/test.sh
-	scripts/docker-integration-tests/coordinator_noop/test.sh
-	scripts/docker-integration-tests/prom_remote_write_backend/test.sh
+  scripts/docker-integration-tests/cold_writes_simple/test.sh
+  # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup.
+  #  scripts/docker-integration-tests/prometheus_replication/test.sh
+
+  scripts/docker-integration-tests/carbon/test.sh
+  scripts/docker-integration-tests/aggregator/test.sh
+  scripts/docker-integration-tests/aggregator_legacy/test.sh
+
+  # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup.
+  #  scripts/docker-integration-tests/query_fanout/test.sh
+
+  scripts/docker-integration-tests/repair/test.sh
+
+  # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup.
+  #  scripts/docker-integration-tests/replication/test.sh
+
+  # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup.
+  #  scripts/docker-integration-tests/multi_cluster_write/test.sh
+
+  scripts/docker-integration-tests/coordinator_config_rules/test.sh
+  scripts/docker-integration-tests/coordinator_noop/test.sh
+  scripts/docker-integration-tests/prom_remote_write_backend/test.sh
 )
 
 # Some systems, including our default Buildkite hosts, don't come with netcat
 # installed and we may not have perms to install it. "Install" it in the worst
 # possible way.
 if ! command -v nc && [[ "$BUILDKITE" == "true" ]]; then
-	echo "installing netcat"
-	NCDIR="$(mktemp -d)"
+  echo "installing netcat"
+  NCDIR="$(mktemp -d)"
 
-	yumdownloader -y --destdir "$NCDIR" --resolve nc
-	(
-		cd "$NCDIR"
-		RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1)
-		rpm2cpio "$RPM" | cpio -id
-	)
+  yumdownloader -y --destdir "$NCDIR" --resolve nc
+  (
+    cd "$NCDIR"
+    RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1)
+    rpm2cpio "$RPM" | cpio -id
+  )
 
-	export PATH="$PATH:$NCDIR/usr/bin"
+  export PATH="$PATH:$NCDIR/usr/bin"
 
-	function cleanup_nc() {
-		rm -rf "$NCDIR"
-	}
+  function cleanup_nc() {
+    rm -rf "$NCDIR"
+  }
 
-	trap cleanup_nc EXIT
+  trap cleanup_nc EXIT
 fi
 
 if [[ -z "$SKIP_SETUP" ]] || [[ "$SKIP_SETUP" == "false" ]]; then
-	scripts/docker-integration-tests/setup.sh
+  scripts/docker-integration-tests/setup.sh
 fi
 
 NUM_TESTS=${#TESTS[@]}
@@ -50,16 +60,16 @@ MAX_IDX=$(((NUM_TESTS*(BUILDKITE_PARALLEL_JOB+1)/BUILDKITE_PARALLEL_JOB_COUNT)-1
 
 ITER=0
 for test in "${TESTS[@]}"; do
-	if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then
-		# Ensure all docker containers have been stopped so we don't run into issues
-		# trying to bind ports.
-		docker rm -f $(docker ps -aq) 2>/dev/null || true
-		echo "----------------------------------------------"
-		echo "running $test"
-		if ! (export M3_PATH=$(pwd) && $test); then
-			echo "--- :bk-status-failed: $test FAILED"
-			exit 1
-		fi
-	fi
-	ITER="$((ITER+1))"
+  if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then
+    # Ensure all docker containers have been stopped so we don't run into issues
+    # trying to bind ports.
+    docker rm -f $(docker ps -aq) 2>/dev/null || true
+    echo "----------------------------------------------"
+    echo "running $test"
+    if ! (export M3_PATH=$(pwd) && $test); then
+      echo "--- :bk-status-failed: $test FAILED"
+      exit 1
+    fi
+  fi
+  ITER="$((ITER+1))"
 done
diff --git a/scripts/docker-integration-tests/setup.sh b/scripts/docker-integration-tests/setup.sh
index 76659a49a2..d90ecb4967 100755
--- a/scripts/docker-integration-tests/setup.sh
+++ b/scripts/docker-integration-tests/setup.sh
@@ -15,8 +15,8 @@ mkdir -p ./bin
 
 # by keeping all the required files in ./bin, it makes the build context
 # for docker much smaller
-cp ./src/query/config/m3coordinator-local-etcd.yml ./bin
-cp ./src/dbnode/config/m3dbnode-local-etcd.yml ./bin
+cp ./src/query/config/m3coordinator-local-docker-etcd.yml ./bin
+cp ./src/dbnode/config/m3dbnode-local-docker-etcd.yml ./bin
 cp ./src/aggregator/config/m3aggregator.yml ./bin
 
 # build images
@@ -26,7 +26,9 @@ function build_image {
   local svc=$1
   echo "creating image for $svc"
   make ${svc}-linux-amd64
-  docker build -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin
+  docker build \
+    --no-cache \
+    -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin
 }
 
 if [[ "$SERVICE" != "" ]]; then
diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml
index bce43d559e..7ed9f8f5ef 100644
--- a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml
+++ b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -22,5 +33,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml
index 7d5cb12598..945b26e3b5 100644
--- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml
+++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml
@@ -13,5 +13,5 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
       useV2BatchAPIs: true
diff --git a/src/dbnode/config/m3dbnode-local-docker-etcd.yml b/src/dbnode/config/m3dbnode-local-docker-etcd.yml
new file mode 100644
index 0000000000..83650bc894
--- /dev/null
+++ b/src/dbnode/config/m3dbnode-local-docker-etcd.yml
@@ -0,0 +1,11 @@
+coordinator: {}
+"db":
+  discovery:
+    "config":
+      "service":
+        "etcdClusters":
+          - "endpoints": ["http://etcd:2379"]
+            "zone": "embedded"
+        "service": "m3db"
+        "zone": "embedded"
+        "env": "default_env"
\ No newline at end of file
diff --git a/src/query/config/m3coordinator-local-docker-etcd.yml b/src/query/config/m3coordinator-local-docker-etcd.yml
new file mode 100644
index 0000000000..c25dff706c
--- /dev/null
+++ b/src/query/config/m3coordinator-local-docker-etcd.yml
@@ -0,0 +1,16 @@
+clusters:
+  - namespaces:
+      - namespace: default
+        type: unaggregated
+        retention: 48h
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - http://etcd:2379