Merge pull request #64 from gruntwork-io/aws-provider-3

Update repo to work with AWS Provider 3.x
gruntwork-io · Sep 13, 2020 · dcc8340 · dcc8340
2 parents 3a1c23b + f65ca00
commit dcc8340
Show file tree

Hide file tree

Showing 24 changed files with 784 additions and 815 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -4,13 +4,13 @@ defaults: &defaults
   machine:
     image: circleci/classic:201711-01
   environment:
-    GRUNTWORK_INSTALLER_VERSION: v0.0.21
-    TERRATEST_LOG_PARSER_VERSION: v0.13.24
-    MODULE_CI_VERSION: v0.13.15
-    TERRAFORM_VERSION: 0.12.2
+    GRUNTWORK_INSTALLER_VERSION: v0.0.29
+    TERRATEST_LOG_PARSER_VERSION: v0.29.0
+    MODULE_CI_VERSION: v0.28.1
+    TERRAFORM_VERSION: 0.12.29
     TERRAGRUNT_VERSION: NONE
     PACKER_VERSION: 1.4.1
-    GOLANG_VERSION: 1.11
+    GOLANG_VERSION: 1.14
 
 version: 2
 jobs:
@@ -26,21 +26,16 @@ jobs:
       - restore_cache:
           keys:
             - v2-external-dep
-            - v2-dep-{{ checksum "test/Gopkg.lock" }}
       # Use the Gruntwork Installer to install the gruntwork-module-circleci-helpers
       - run: curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}"
       - run: gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}"
       - run: gruntwork-install --binary-name "terratest_log_parser" --repo "https://github.com/gruntwork-io/terratest" --tag "${TERRATEST_LOG_PARSER_VERSION}"
-      - run: configure-environment-for-gruntwork-module --circle-ci-2-machine-executor --terraform-version ${TERRAFORM_VERSION} --terragrunt-version ${TERRAGRUNT_VERSION} --packer-version ${PACKER_VERSION} --use-go-dep --go-version ${GOLANG_VERSION} --go-src-path test
+      - run: configure-environment-for-gruntwork-module --terraform-version ${TERRAFORM_VERSION} --terragrunt-version ${TERRAGRUNT_VERSION} --packer-version ${PACKER_VERSION} --go-version ${GOLANG_VERSION}
       - save_cache:
           key: v2-external-dep
           paths:
             - $HOME/terraform
             - $HOME/packer
-      - save_cache:
-          key: v2-dep-{{ checksum "test/Gopkg.lock" }}
-          paths:
-            - $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/test/vendor
 
       # Install external dependencies
       - run: DEBIAN_FRONTEND=noninteractive sudo apt-get update && sudo apt-get install -y python-pip
@@ -83,7 +78,7 @@ jobs:
       - run: gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}"
       - run: gruntwork-install --module-name "build-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}"
       - run: gruntwork-install --module-name "aws-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}"
-      - run: configure-environment-for-gruntwork-module --circle-ci-2-machine-executor --go-src-path test --use-go-dep --terraform-version NONE --terragrunt-version NONE --glide-version NONE
+      - run: configure-environment-for-gruntwork-module --terraform-version NONE --terragrunt-version NONE
 
       - run: ~/project/.circleci/publish-amis.sh "ubuntu-ami"
       - run: ~/project/.circleci/publish-amis.sh "amazon-linux-ami"

diff --git a/examples/couchbase-ami/couchbase.json b/examples/couchbase-ami/couchbase.json
@@ -105,7 +105,7 @@
     "pause_before": "5s",
     "inline": [
       "apt-get update",
-      "DEBIAN_FRONTEND=noninteractive apt-get install -y git"
+      "DEBIAN_FRONTEND=noninteractive apt-get install -y git lsb-release"
     ],
     "only": ["ubuntu-docker", "ubuntu-18-docker"]
   },{
@@ -141,7 +141,7 @@
     "destination": "/tmp/terraform-aws-couchbase/modules"
   },{
     "type": "file",
-    "source": "{{template_dir}}/sync_gateway.json",
+    "source": "{{template_dir}}/sync_gateway_{{user `edition`}}.json",
     "destination": "/tmp/terraform-aws-couchbase/sync_gateway.json"
   },{
     "type": "shell",

diff --git a/examples/couchbase-ami/sync_gateway_community.json b/examples/couchbase-ami/sync_gateway_community.json
@@ -0,0 +1,27 @@
+{
+  "adminInterface": "<ADMIN_INTERFACE>",
+  "interface": "<INTERFACE>",
+  "databases": {
+    "<DB_NAME>": {
+      "server": "<SERVERS>",
+      "username": "<DB_USERNAME>",
+      "password": "<DB_PASSWORD>",
+      "bucket": "<BUCKET_NAME>",
+      "num_index_replicas": 0,
+      "users": {"GUEST": {"disabled": false, "admin_channels": ["*"]}}
+    }
+  },
+  "logging": {
+    "default": {
+      "logFilePath": "/home/sync_gateway/logs/sync-gateway.log",
+      "logKeys": ["*"],
+      "logLevel": "info",
+      "rotation": {
+        "maxsize": 100,
+        "maxage": 30,
+        "maxbackups": 5,
+        "localtime": true
+      }
+    }
+  }
+}
diff --git a/examples/couchbase-ami/sync_gateway.json → ...ouchbase-ami/sync_gateway_enterprise.json b/examples/couchbase-ami/sync_gateway.json → ...ouchbase-ami/sync_gateway_enterprise.json
@@ -23,4 +23,4 @@
       }
     }
   }
-}
+}
diff --git a/examples/couchbase-cluster-mds/user-data/user-data-couchbase-data-nodes.sh b/examples/couchbase-cluster-mds/user-data/user-data-couchbase-data-nodes.sh
@@ -60,7 +60,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create RBAC user $user_name" \
-    "SUCCESS: RBAC user set" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "user-manage" \
@@ -78,7 +78,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create bucket $bucket_name" \
-    "SUCCESS: Bucket created" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "bucket-create" \

diff --git a/examples/couchbase-cluster-simple-dns-tls/user-data/user-data.sh b/examples/couchbase-cluster-simple-dns-tls/user-data/user-data.sh
@@ -56,7 +56,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create RBAC user $user_name" \
-    "SUCCESS: RBAC user set" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "user-manage" \
@@ -74,7 +74,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create bucket $bucket_name" \
-    "SUCCESS: Bucket created" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "bucket-create" \

diff --git a/examples/couchbase-cluster-simple/user-data/user-data.sh b/examples/couchbase-cluster-simple/user-data/user-data.sh
@@ -57,7 +57,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create RBAC user $user_name" \
-    "SUCCESS: RBAC user set" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "user-manage" \
@@ -75,7 +75,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create bucket $bucket_name" \
-    "SUCCESS: Bucket created" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "bucket-create" \

diff --git a/examples/couchbase-multi-datacenter-replication/user-data/user-data-primary.sh b/examples/couchbase-multi-datacenter-replication/user-data/user-data-primary.sh
@@ -42,7 +42,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create RBAC user $user_name" \
-    "SUCCESS: RBAC user set" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "user-manage" \
@@ -60,7 +60,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create bucket $bucket_name" \
-    "SUCCESS: Bucket created" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "bucket-create" \

diff --git a/examples/couchbase-multi-datacenter-replication/user-data/user-data-replica.sh b/examples/couchbase-multi-datacenter-replication/user-data/user-data-replica.sh
@@ -42,7 +42,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create RBAC user $user_name" \
-    "SUCCESS: RBAC user set" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "user-manage" \
@@ -60,7 +60,7 @@ function create_test_resources {
 
   run_couchbase_cli_with_retry \
     "Create bucket $bucket_name" \
-    "SUCCESS: Bucket created" \
+    "SUCCESS:" \
     "$max_retries" \
     "$sleep_between_retries_sec" \
     "bucket-create" \

diff --git a/modules/couchbase-cluster/main.tf b/modules/couchbase-cluster/main.tf
@@ -40,6 +40,12 @@ resource "aws_autoscaling_group" "autoscaling_group" {
       propagate_at_launch = tag.value["propagate_at_launch"]
     }
   }
+
+  # When using `aws_autoscaling_attachment` resources to manage target group and loadbalancer attachments, we need to
+  # ignore changes to these properties as they are updated in the cloud and not terraform.
+  lifecycle {
+    ignore_changes = [load_balancers, target_group_arns]
+  }
 }
 
 # ---------------------------------------------------------------------------------------------------------------------

diff --git a/modules/install-couchbase-server/install-couchbase-server b/modules/install-couchbase-server/install-couchbase-server
@@ -20,13 +20,17 @@ readonly ENTERPRISE_EDITION="enterprise"
 readonly COMMUNITY_EDITION="community"
 readonly DEFAULT_EDITION="$ENTERPRISE_EDITION"
 
-readonly DEFAULT_COUCHBASE_ENTERPRISE_VERSION="5.1.1"
-readonly DEFAULT_COUCHBASE_ENTERPRISE_SHA256_CHECKSUM_UBUNTU_16_04="058acc6567db7acd8dcb80aa55a7a1de1b318848255bcb2878b285c0e52ecc8c"
-readonly DEFAULT_COUCHBASE_ENTERPRISE_SHA256_CHECKSUM_AMAZON_LINUX="0360be31f20709af2a74f602ad7c81fbfd8f40fe9bf671b167f1cc6260cdcc38"
-
-readonly DEFAULT_COUCHBASE_COMMUNITY_VERSION="5.0.1"
-readonly DEFAULT_COUCHBASE_COMMUNITY_MD5_CHECKSUM_UBUNTU_16_04="efd890ccaea2a83020c11746315a404f"
-readonly DEFAULT_COUCHBASE_COMMUNITY_MD5_CHECKSUM_AMAZON_LINUX="1c2bfda713f939911494b31727d7eb30"
+# Get these from the downloads section of the Couchbase website: https://www.couchbase.com/downloads
+readonly DEFAULT_COUCHBASE_ENTERPRISE_VERSION="6.6.0"
+readonly DEFAULT_COUCHBASE_ENTERPRISE_CHECKSUM_UBUNTU_18_04="8e7fd5434537094be2fbdfedf3ab5005f0f7d5b9d0578f59ce540b424215b728"
+readonly DEFAULT_COUCHBASE_ENTERPRISE_CHECKSUM_UBUNTU_16_04="9f666b2e39c11b17a9cc74c00967d97efeab08e23b93e8bbdec582ce009c65c9"
+readonly DEFAULT_COUCHBASE_ENTERPRISE_CHECKSUM_AMAZON_LINUX="91fd6bc72db44e59a12a2945944a698f782b621cf757ab95248f831a16b2c73a"
+
+# Get these from the downloads section of the Couchbase website: https://www.couchbase.com/downloads
+readonly DEFAULT_COUCHBASE_COMMUNITY_VERSION="6.5.1"
+readonly DEFAULT_COUCHBASE_COMMUNITY_CHECKSUM_UBUNTU_18_04="c4951cdab01759020444e4648023721ae3a333257591252475d34d5fc6ac8857"
+readonly DEFAULT_COUCHBASE_COMMUNITY_CHECKSUM_UBUNTU_16_04="baf65fb9cbcec87783d4e9c3ec067143a42cdeef13a884e1f917e8d2f14044b7"
+readonly DEFAULT_COUCHBASE_COMMUNITY_CHECKSUM_AMAZON_LINUX="93a99222c2df857ebc08e60e85b5a3378b63d574ed24f33066542934ce60bf74"
 
 readonly SHA256_CHECKSUM_TYPE="sha256"
 readonly MD5_CHECKSUM_TYPE="md5"
@@ -81,7 +85,7 @@ function install_couchbase_on_ubuntu {
 
   log_info "Installing Couchbase dependencies"
   sudo DEBIAN_FRONTEND=noninteractive apt-get update
-  sudo apt-get install -y python-httplib2
+  sudo apt-get install -y python-httplib2 bzip2
 
   log_info "Downloading Couchbase from $url to $filepath"
   if [[ "$(curl --location --silent --fail --show-error -o "$dpkg_name" -w "%{http_code}" "$url")" == "404" ]]; then
@@ -109,12 +113,12 @@ function install_couchbase_on_amazon_linux {
 
   log_info "Installing Couchbase $version ($edition edition) on Amazon Linux"
 
-  local readonly filepath="couchbase-server-${edition}-${version}-centos6.x86_64.rpm"
+  local readonly filepath="couchbase-server-${edition}-${version}-centos7.x86_64.rpm"
   local readonly url="https://packages.couchbase.com/releases/$version/$filepath"
 
   log_info "Installing Couchbase dependencies"
   sudo yum update -y
-  sudo yum install -y openssl
+  sudo yum install -y openssl bzip2
 
   log_info "Downloading Couchbase from $url to $filepath"
   curl --location --silent --fail --show-error -O "$url"
@@ -248,12 +252,24 @@ function install {
   if [[ -z "$version" ]]; then
     if [[ "$edition" == "$COMMUNITY_EDITION" ]]; then
       version="$DEFAULT_COUCHBASE_COMMUNITY_VERSION"
-      checksum=$(os_is_amazon_linux && echo -n "$DEFAULT_COUCHBASE_COMMUNITY_MD5_CHECKSUM_AMAZON_LINUX" || echo -n "$DEFAULT_COUCHBASE_COMMUNITY_MD5_CHECKSUM_UBUNTU_16_04")
-      checksum_type="$MD5_CHECKSUM_TYPE"
+      checksum_type="$SHA256_CHECKSUM_TYPE"
+      if os_is_ubuntu "16.04"; then
+        checksum="$DEFAULT_COUCHBASE_COMMUNITY_CHECKSUM_UBUNTU_16_04"
+      elif os_is_ubuntu "18.04"; then
+        checksum="$DEFAULT_COUCHBASE_COMMUNITY_CHECKSUM_UBUNTU_18_04"
+      else
+        checksum="$DEFAULT_COUCHBASE_COMMUNITY_CHECKSUM_AMAZON_LINUX"
+      fi
     else
       version="$DEFAULT_COUCHBASE_ENTERPRISE_VERSION"
-      checksum=$(os_is_amazon_linux && echo -n "$DEFAULT_COUCHBASE_ENTERPRISE_SHA256_CHECKSUM_AMAZON_LINUX" || echo -n "$DEFAULT_COUCHBASE_ENTERPRISE_SHA256_CHECKSUM_UBUNTU_16_04")
       checksum_type="$SHA256_CHECKSUM_TYPE"
+      if os_is_ubuntu "16.04"; then
+        checksum="$DEFAULT_COUCHBASE_ENTERPRISE_CHECKSUM_UBUNTU_16_04"
+      elif os_is_ubuntu "18.04"; then
+        checksum="$DEFAULT_COUCHBASE_ENTERPRISE_CHECKSUM_UBUNTU_18_04"
+      else
+        checksum="$DEFAULT_COUCHBASE_ENTERPRISE_CHECKSUM_AMAZON_LINUX"
+      fi
     fi
   fi
 
@@ -277,4 +293,3 @@ function install {
 }
 
 install "$@"
-
diff --git a/modules/install-sync-gateway/install-sync-gateway b/modules/install-sync-gateway/install-sync-gateway
@@ -19,13 +19,15 @@ readonly ENTERPRISE_EDITION="enterprise"
 readonly COMMUNITY_EDITION="community"
 readonly DEFAULT_EDITION="$ENTERPRISE_EDITION"
 
-readonly DEFAULT_SYNC_GATEWAY_ENTERPRISE_VERSION="2.0.0"
-readonly DEFAULT_SYNC_GATEWAY_ENTERPRISE_MD5_CHECKSUM_UBUNTU_16_04="a11cf149a7f64acb7368e1a18c85ebe7"
-readonly DEFAULT_SYNC_GATEWAY_ENTERPRISE_MD5_CHECKSUM_AMAZON_LINUX="d2f77e426e299fa08857e4cf7dbe4d77"
+# Get these from the downloads section of the Couchbase website: https://www.couchbase.com/downloads
+readonly DEFAULT_SYNC_GATEWAY_ENTERPRISE_VERSION="2.7.3"
+readonly DEFAULT_SYNC_GATEWAY_ENTERPRISE_CHECKSUM_UBUNTU="64868af179f4cd4aaf753054dc24c246d05e2df14aec80af591f0ffbb41f456b"
+readonly DEFAULT_SYNC_GATEWAY_ENTERPRISE_CHECKSUM_AMAZON_LINUX="be7da0c542b6eab3bea7bf56acfc7fb104c712b2d861077e6f30842e066617ef"
 
-readonly DEFAULT_SYNC_GATEWAY_COMMUNITY_VERSION="2.0.0"
-readonly DEFAULT_SYNC_GATEWAY_COMMUNITY_MD5_CHECKSUM_UBUNTU_16_04="9549ae33868e21f94bf2a6ef73e8eb0b"
-readonly DEFAULT_SYNC_GATEWAY_COMMUNITY_MD5_CHECKSUM_AMAZON_LINUX="a4a6e56b6ea5444e723a001eb5bad04a"
+# Get these from the downloads section of the Couchbase website: https://www.couchbase.com/downloads
+readonly DEFAULT_SYNC_GATEWAY_COMMUNITY_VERSION="2.7.3"
+readonly DEFAULT_SYNC_GATEWAY_COMMUNITY_CHECKSUM_UBUNTU="4e0306d06840c8e5a6df6984cb47feb5706df5a11058e4253eaab9d9a9044d18"
+readonly DEFAULT_SYNC_GATEWAY_COMMUNITY_CHECKSUM_AMAZON_LINUX="068a52128ed92b4781bd07a2ca962d44f2e13c7c1ba45071af233913fe5fd40e"
 
 readonly SHA256_CHECKSUM_TYPE="sha256"
 readonly MD5_CHECKSUM_TYPE="md5"
@@ -111,17 +113,17 @@ function install_sync_gateway_on_amazon_linux {
   log_info "Installing SyncGateway from $filepath"
   sudo rpm --install "$filepath"
 
-  # These commands sometimes exit with an error for no obvious reason, so we just ignore the error...
+  # These commands sometimes fail and exit with an error for no obvious reason. Moreover, it appears that the
+  # sync_gateway process can't be stopped cleanly using service, so we subsequently use pkill.
   set +e
   sudo chkconfig sync_gateway off
   sudo service sync_gateway stop
+  sudo pkill sync_gateway
   set -e
 
-  # Remove the original log paths to reduce confusion
-  sudo rm -rf /var/log/sync_gateway
-
   # Create systemd unit to run Sync Gateway. The installer creates an init.d script, but that doesn't work correctly
-  # in Docker, and it doesn't auto-restart Sync Gateway if it crashes, so this is a more effective option.
+  # in Docker, and it doesn't auto-restart Sync Gateway if it crashes, so this is a more
+  # effective option.
   log_info "Creating systemd unit for Sync Gateway in $DEFAULT_SYNC_GATEWAY_SYSTEMD_UNIT_PATH."
   sudo tee "$DEFAULT_SYNC_GATEWAY_SYSTEMD_UNIT_PATH" > /dev/null << EOF
 [Unit]
@@ -153,6 +155,9 @@ TimeoutSec=60
 WantedBy=multi-user.target
 EOF
 
+  # Remove the original log paths to reduce confusion
+  sudo rm -rf /var/log/sync_gateway
+
   log_info "Cleaning up $filepath"
   rm -f "$filepath"
 }
@@ -246,12 +251,12 @@ function install {
   if [[ -z "$version" ]]; then
     if [[ "$edition" == "$COMMUNITY_EDITION" ]]; then
       version="$DEFAULT_SYNC_GATEWAY_COMMUNITY_VERSION"
-      checksum=$(os_is_amazon_linux && echo -n "$DEFAULT_SYNC_GATEWAY_COMMUNITY_MD5_CHECKSUM_AMAZON_LINUX" || echo -n "$DEFAULT_SYNC_GATEWAY_COMMUNITY_MD5_CHECKSUM_UBUNTU_16_04")
-      checksum_type="$MD5_CHECKSUM_TYPE"
+      checksum=$(os_is_amazon_linux && echo -n "$DEFAULT_SYNC_GATEWAY_COMMUNITY_CHECKSUM_AMAZON_LINUX" || echo -n "$DEFAULT_SYNC_GATEWAY_COMMUNITY_CHECKSUM_UBUNTU")
+      checksum_type="$SHA256_CHECKSUM_TYPE"
     else
       version="$DEFAULT_SYNC_GATEWAY_ENTERPRISE_VERSION"
-      checksum=$(os_is_amazon_linux && echo -n "$DEFAULT_SYNC_GATEWAY_ENTERPRISE_MD5_CHECKSUM_AMAZON_LINUX" || echo -n "$DEFAULT_SYNC_GATEWAY_ENTERPRISE_MD5_CHECKSUM_UBUNTU_16_04")
-      checksum_type="$MD5_CHECKSUM_TYPE"
+      checksum=$(os_is_amazon_linux && echo -n "$DEFAULT_SYNC_GATEWAY_ENTERPRISE_CHECKSUM_AMAZON_LINUX" || echo -n "$DEFAULT_SYNC_GATEWAY_ENTERPRISE_CHECKSUM_UBUNTU")
+      checksum_type="$SHA256_CHECKSUM_TYPE"
     fi
   fi
 
@@ -274,4 +279,3 @@ function install {
 }
 
 install "$@"
-
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,4 +23,4 @@ @@
           }
         }
       }
-    }
+    }