diff --git a/.gitignore b/.gitignore index 6d79455..85622a3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ VULTR_SSH_PRIVATE_KEY VULTR_SSH_PUBLIC_KEY vultr_*.json tfplan-destroy +*.pem \ No newline at end of file diff --git a/main.tf b/main.tf index b59a815..eac3f84 100644 --- a/main.tf +++ b/main.tf @@ -44,7 +44,7 @@ locals { } } data "template_file" "mongodb_user_data" { - template = file("install-server.sh") + template = file("user-data/default-server.sh") vars = merge(local.instance_user_data, { environment_type = "toolbox" @@ -69,8 +69,32 @@ module "das_apt_repository" { create_resource = true name = "das-apt-repository" environment = local.environment - user_data_file = file("install-apt-repository.sh") + user_data_file = file("user-data/deb-package-server.sh") ssh_key_ids = var.ssh_key_ids region = var.region plan = "vc2-1c-2gb" } + +module "mongodb_shard" { + source = "./mongodb-shards" + region = var.region + environment = local.environment + + shard = { + clusters = 1 + nodes_per_cluster = 2, + instance_type = "vc2-1c-2gb" + } + + config_set = { + clusters = 1 + nodes_per_cluster = 2, + instance_type = "vc2-1c-2gb" + } + + mongos = { + nodes = 1 + instance_type = "vc2-1c-2gb" + } + +} diff --git a/mongodb-shards/config-set.tf b/mongodb-shards/config-set.tf new file mode 100644 index 0000000..66f8e44 --- /dev/null +++ b/mongodb-shards/config-set.tf @@ -0,0 +1,50 @@ +module "mongodb_cluster_config_set" { + source = "../instance" + create_resource = true + count = var.config_set.clusters * var.config_set.nodes_per_cluster + name = "configrepl${count.index}-${random_string.random[count.index].id}" + environment = var.environment + user_data_file = file("mongodb-shards/user-data/config-set.sh") + ssh_key_ids = [vultr_ssh_key.mongodb_ssh_key.id] + region = var.region + plan = var.config_set.instance_type +} + +resource "null_resource" "mongo_init_config_set" { + depends_on = [ + module.mongodb_cluster_config_set + ] + + count = var.config_set.clusters + triggers = { + cluster_instance_ids = join(",", module.mongodb_cluster_config_set[*].instance_ip) + } + + provisioner "file" { + source = "mongodb-shards/user-data/init-config-set.sh" + destination = "/tmp/bootstrap-cluster.sh" + + + connection { + type = "ssh" + user = "root" + private_key = file(local_file.mongodb_private_key.filename) + host = slice(module.mongodb_cluster_config_set[*].instance_ip, count.index * var.config_set.nodes_per_cluster, (count.index + 1) * var.config_set.nodes_per_cluster)[0] + } + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /tmp/bootstrap-cluster.sh", + "/tmp/bootstrap-cluster.sh ${join(" ", + slice(module.mongodb_cluster_config_set[*].instance_ip, count.index * var.config_set.nodes_per_cluster, (count.index + 1) * var.config_set.nodes_per_cluster))}" + ] + + connection { + type = "ssh" + user = "root" + private_key = file(local_file.mongodb_private_key.filename) + host = slice(module.mongodb_cluster_config_set[*].instance_ip, count.index * var.config_set.nodes_per_cluster, (count.index + 1) * var.config_set.nodes_per_cluster)[0] + } + } +} diff --git a/mongodb-shards/main.tf b/mongodb-shards/main.tf new file mode 100644 index 0000000..766bb39 --- /dev/null +++ b/mongodb-shards/main.tf @@ -0,0 +1,62 @@ +variable "region" { + type = string +} + +variable "environment" { + type = string +} + +variable "shard" { + type = object({ + clusters = number + nodes_per_cluster = number + instance_type = string + }) +} + + +variable "config_set" { + type = object({ + clusters = number + nodes_per_cluster = number + instance_type = string + }) +} + +variable "mongos" { + type = object({ + nodes = number + instance_type = string + }) +} + +terraform { + required_providers { + vultr = { + source = "vultr/vultr" + version = "2.15.1" + } + } +} + +resource "random_string" "random" { + length = 16 + special = false + count = var.config_set.clusters * var.config_set.nodes_per_cluster +} + +resource "tls_private_key" "mongodb_ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "vultr_ssh_key" "mongodb_ssh_key" { + name = "mongodb-shard-${formatdate("YYYYMMDDhhmmss", timestamp())}" + ssh_key = tls_private_key.mongodb_ssh_key.public_key_openssh +} + +resource "local_file" "mongodb_private_key" { + content = tls_private_key.mongodb_ssh_key.private_key_pem + filename = "${vultr_ssh_key.mongodb_ssh_key.name}.pem" +} + diff --git a/mongodb-shards/mongos.tf b/mongodb-shards/mongos.tf new file mode 100644 index 0000000..f8880ab --- /dev/null +++ b/mongodb-shards/mongos.tf @@ -0,0 +1,57 @@ +module "mongodb_cluster_mongos" { + source = "../instance" + create_resource = true + count = var.mongos.nodes + name = "mongos${count.index}-${random_string.random[count.index].id}" + environment = var.environment + user_data_file = file("mongodb-shards/user-data/config-mongos.sh") + ssh_key_ids = [vultr_ssh_key.mongodb_ssh_key.id] + region = var.region + plan = var.mongos.instance_type + + depends_on = [ + module.mongodb_cluster_config_set, + module.mongodb_cluster_shard, + null_resource.mongo_init_config_set, + null_resource.mongo_init_shard, + ] +} + + +resource "null_resource" "mongo_init_mongos" { + depends_on = [ + module.mongodb_cluster_mongos + ] + + count = var.mongos.nodes + triggers = { + cluster_instance_ids = join(",", module.mongodb_cluster_mongos[*].instance_ip) + } + + provisioner "file" { + source = "mongodb-shards/user-data/init-mongos.sh" + destination = "/tmp/bootstrap-cluster.sh" + + + connection { + type = "ssh" + user = "root" + private_key = file(local_file.mongodb_private_key.filename) + host = module.mongodb_cluster_mongos[count.index].instance_ip + } + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /tmp/bootstrap-cluster.sh", + "/tmp/bootstrap-cluster.sh --config-set ${join(",", module.mongodb_cluster_config_set[*].instance_ip)} --shards ${join(",", module.mongodb_cluster_shard[*].instance_ip)} --shard-clusters ${var.config_set.clusters}" + ] + + connection { + type = "ssh" + user = "root" + private_key = file(local_file.mongodb_private_key.filename) + host = module.mongodb_cluster_mongos[count.index].instance_ip + } + } +} diff --git a/mongodb-shards/shard.tf b/mongodb-shards/shard.tf new file mode 100644 index 0000000..f636cb5 --- /dev/null +++ b/mongodb-shards/shard.tf @@ -0,0 +1,55 @@ +module "mongodb_cluster_shard" { + source = "../instance" + create_resource = true + count = var.shard.nodes_per_cluster * var.shard.clusters + name = "shard${count.index % var.shard.nodes_per_cluster == 0 ? var.shard.nodes_per_cluster : count.index % var.shard.nodes_per_cluster}node${count.index}-${random_string.random[count.index].id}" + environment = var.environment + user_data_file = file("mongodb-shards/user-data/config-shard.sh") + ssh_key_ids = [vultr_ssh_key.mongodb_ssh_key.id] + region = var.region + plan = var.shard.instance_type + + depends_on = [ + module.mongodb_cluster_config_set, + null_resource.mongo_init_config_set + ] +} + +resource "null_resource" "mongo_init_shard" { + depends_on = [ + module.mongodb_cluster_shard + ] + + count = var.shard.clusters + triggers = { + cluster_instance_ids = join(",", module.mongodb_cluster_shard[*].instance_ip) + } + + provisioner "file" { + source = "mongodb-shards/user-data/init-shard-set.sh" + destination = "/tmp/bootstrap-cluster.sh" + + + connection { + type = "ssh" + user = "root" + private_key = file(local_file.mongodb_private_key.filename) + host = slice(module.mongodb_cluster_shard[*].instance_ip, count.index * var.shard.nodes_per_cluster, (count.index + 1) * var.shard.nodes_per_cluster)[0] + } + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /tmp/bootstrap-cluster.sh", var.shard.clusters, + "/tmp/bootstrap-cluster.sh ${join(" ", + slice(module.mongodb_cluster_shard[*].instance_ip, count.index * var.shard.nodes_per_cluster, (count.index + 1) * var.shard.nodes_per_cluster))}" + ] + + connection { + type = "ssh" + user = "root" + private_key = file(local_file.mongodb_private_key.filename) + host = slice(module.mongodb_cluster_shard[*].instance_ip, count.index * var.shard.nodes_per_cluster, (count.index + 1) * var.shard.nodes_per_cluster)[0] + } + } +} diff --git a/mongodb-shards/user-data/config-mongos.sh b/mongodb-shards/user-data/config-mongos.sh new file mode 100755 index 0000000..1befaae --- /dev/null +++ b/mongodb-shards/user-data/config-mongos.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Global Variables +PID_FILE="/run/terraform-mongos-manager.pid" +CURRENT_SCRIPT_LOGS="/tmp/mongos-setup.log" + +exec >$CURRENT_SCRIPT_LOGS 2>&1 + +function raise_command_not_found() { + local cmd="$1" + + if ! command -v "$cmd" &> /dev/null; then + echo "$cmd is required" + exit 1 + fi +} + +function enable_firewall() { + if ! command -v ufw &>/dev/null; then + apt-get update + apt-get install ufw + fi + + ufw enable + ufw allow ssh + ufw allow 28041 +} + +function cleanup() { + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi +} + +function create_pid_file() { + if [ -f "$PID_FILE" ]; then + echo "Script is already running with PID $(cat "$PID_FILE"). Waiting process to end..." + exit 1 + fi + + echo $$ > "$PID_FILE" + trap cleanup EXIT +} + +function install_mongodb() { + if command -v mongod; then + echo "Skipping mongodb installation because it's already installed." + return 0 + fi + + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + apt update + apt install -y mongodb-org + systemctl start mongod + systemctl enable mongod + systemctl status mongod + + raise_command_not_found "mongod" +} + +create_pid_file +install_mongodb +enable_firewall diff --git a/mongodb-shards/user-data/config-set.sh b/mongodb-shards/user-data/config-set.sh new file mode 100755 index 0000000..13b1b72 --- /dev/null +++ b/mongodb-shards/user-data/config-set.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# GLOBAL VARIABLES +PID_FILE="/run/terraform-shard-manager.pid" +MONGODB_DB_CLUSTER="/opt/shard/configsrv-node" +MONGODB_CLUSTER_CONFIG="/opt/shard/mongod.conf" +MONGODB_LOGS="/tmp/mongod.log" +CURRENT_SCRIPT_LOGS="/tmp/config-set.log" + +exec >$CURRENT_SCRIPT_LOGS 2>&1 + + +function raise_command_not_found() { + local cmd="$1" + + if ! command -v "$cmd" &> /dev/null; then + echo "$cmd is required" + exit 1 +fi +} + + +function cleanup() { + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi +} + +function create_pid_file() { + if [ -f "$PID_FILE" ]; then + echo "Script is already running with PID $(cat "$PID_FILE"). Waiting process to end..." + exit 1 + fi + + echo $$ > "$PID_FILE" + trap cleanup EXIT +} + +function enable_firewall() { + if ! command -v ufw &>/dev/null; then + apt-get update + apt-get install ufw + fi + + ufw enable + ufw allow ssh + ufw allow 28041 +} + +function install_mongodb() { + if command -v mongod; then + echo "Skipping mongodb installation because it's already installed." + return 0 + fi + + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + apt update + apt install -y mongodb-org + systemctl start mongod + systemctl enable mongod + systemctl status mongod + + raise_command_not_found "mongod" +} + +function setup() { + local public_ip + + raise_command_not_found "curl" + + public_ip="$(curl ipinfo.io/ip)" + + mkdir -p $MONGODB_DB_CLUSTER + + cat < $MONGODB_CLUSTER_CONFIG +systemLog: + destination: file + path: "$MONGODB_LOGS" + logAppend: true +storage: + dbPath: "$MONGODB_DB_CLUSTER" +net: + port: 28041 + bindIp: "$public_ip,localhost" +replication: + replSetName: "config_repl" +sharding: + clusterRole: "configsvr" +EOF + + mongod --config $MONGODB_CLUSTER_CONFIG & +} + +create_pid_file +install_mongodb +enable_firewall +setup diff --git a/mongodb-shards/user-data/config-shard.sh b/mongodb-shards/user-data/config-shard.sh new file mode 100755 index 0000000..90d2b72 --- /dev/null +++ b/mongodb-shards/user-data/config-shard.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# GLOBAL VARIABLES +PID_FILE="/run/terraform-shard-manager.pid" +MONGODB_DB_CLUSTER="/opt/shard/shardsvr-node" +MONGODB_CLUSTER_CONFIG="/opt/shard/mongod.conf" +MONGODB_LOGS="/tmp/mongod.log" +CURRENT_SCRIPT_LOGS="/tmp/config-shard.log" + +exec >$CURRENT_SCRIPT_LOGS 2>&1 + + +function raise_command_not_found() { + local cmd="$1" + + if ! command -v "$cmd" &> /dev/null; then + echo "$cmd is required" + exit 1 +fi +} + + +function cleanup() { + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi +} + +function create_pid_file() { + if [ -f "$PID_FILE" ]; then + echo "Script is already running with PID $(cat "$PID_FILE"). Waiting process to end..." + exit 1 + fi + + echo $$ > "$PID_FILE" + trap cleanup EXIT +} + +function enable_firewall() { + if ! command -v ufw &>/dev/null; then + apt-get update + apt-get install ufw + fi + + ufw enable + ufw allow ssh + ufw allow 28041 +} + +function install_mongodb() { + if command -v mongod; then + echo "Skipping mongodb installation because it's already installed." + return 0 + fi + + wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list + apt update + apt install -y mongodb-org + systemctl start mongod + systemctl enable mongod + systemctl status mongod + + raise_command_not_found "mongod" +} + +function setup() { + local public_ip + + raise_command_not_found "curl" + + public_ip="$(curl ipinfo.io/ip)" + + mkdir -p $MONGODB_DB_CLUSTER + + cat < $MONGODB_CLUSTER_CONFIG +systemLog: + destination: file + path: "$MONGODB_LOGS" + logAppend: true +storage: + dbPath: "$MONGODB_DB_CLUSTER" +net: + port: 28041 + bindIp: "$public_ip,localhost" +replication: + replSetName: "shard_repl" +sharding: + clusterRole: "shardsvr" +EOF + + mongod --config $MONGODB_CLUSTER_CONFIG & +} + +create_pid_file +install_mongodb +enable_firewall +setup diff --git a/mongodb-shards/user-data/init-config-set.sh b/mongodb-shards/user-data/init-config-set.sh new file mode 100755 index 0000000..edee19f --- /dev/null +++ b/mongodb-shards/user-data/init-config-set.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# GLOBAL VARIABLES +PID_FILE="/run/terraform-shard-manager.pid" +CURRENT_SCRIPT_LOGS="/tmp/bootstrap-cluster.log" + +exec >$CURRENT_SCRIPT_LOGS 2>&1 + +function cleanup() { + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi +} + +function create_pid_file() { + local max_attempts=5 + + for ((i=$max_attempts; i>0;i--)); do + if [ -f "$PID_FILE" ]; then + echo "Script is already running with PID $(cat "$PID_FILE"). Waiting process to end..." + sleep 1m + continue + fi + + echo $$ > "$PID_FILE" + trap cleanup EXIT + return 0 + done + + echo "Exiting due to waiting too long for the process to finish." + exit 1 +} + +function get_replica_config() { + local public_ips=("$@") + local members="[]" + local config='{"_id": "config_repl", "members": []}' + + for ((i=0; i<${#public_ips[@]}; i++)) { + local member="{\"_id\": $i, \"host\": \"${public_ips[i]}:28041\"}" + members=$(jq --argjson newMember "$member" '. + [$newMember]' <<<"$members") + } + + config=$(jq --argjson members "$members" '.members += $members' <<<"$config") + + echo "$config" +} + +function get_replica_config_script() { + local rsconf + local replica_set_config + rsconf=$(get_replica_config "$@") + replica_set_config=$(cat <0;i--)); do + if [ -f "$PID_FILE" ]; then + echo "Script is already running with PID $(cat "$PID_FILE"). Waiting process to end..." + sleep 1m + continue + fi + + echo $$ > "$PID_FILE" + trap cleanup EXIT + return 0 + done + + echo "Exiting due to waiting too long for the process to finish." + exit 1 +} + +function cluster_initialized() { + if [[ "$(mongosh --port 28041 --eval 'rs.status().ok')" == "1" ]]; then + return 0 + fi + + return 1 +} + +function add_port_to_ips() { + local ips_string="$1" + local port="$2" + local ips_with_ports=() + + IFS=',' read -r -a ip_array <<< "$ips_string" + + for ip in "${ip_array[@]}"; do + ips_with_ports+=("${ip}:${port}") + done + + IFS=','; echo "${ips_with_ports[*]}" +} + +function setup() { + local public_ip + + if cluster_initialized; then + echo "The cluster has already been initialized. Skipping initialization..." + exit 0 + fi + + public_ip="$(curl ipinfo.io/ip)" + + mkdir -p $MONGODB_DB_CLUSTER + + cat < $MONGODB_CLUSTER_CONFIG +systemLog: + destination: file + path: "$MONGODB_LOGS" + logAppend: true +net: + port: 28041 + bindIp: "$public_ip,localhost" +sharding: + configDB: "config_repl/$CONFIG_SET" +EOF + + mongos --port 28041 --config $MONGODB_CLUSTER_CONFIG & + + sleep 2m + + IFS=',' read -r -a shards <<< "$SHARDS" + add_shards "${shards[@]}" +} + +function add_shards() { + local shards=("$@") + + for shard in "${shards[@]}"; do + echo "Adding shard: $shard" + mongosh --port 28041 --eval "sh.addShard('$shard')" + done +} + +function parse_args() { + while [[ "$#" -gt 0 ]]; do + case $1 in + --config-set) + CONFIG_SET=$(add_port_to_ips "$2" "28041") + shift 2 + ;; + --shards) + SHARDS=$(add_port_to_ips "$2" "28041") + shift 2 + ;; + *) + echo "Unknown parameter: $1" + exit 1 + ;; + esac + done +} + +create_pid_file +parse_args "$@" +setup \ No newline at end of file diff --git a/mongodb-shards/user-data/init-shard-set.sh b/mongodb-shards/user-data/init-shard-set.sh new file mode 100755 index 0000000..3a796bd --- /dev/null +++ b/mongodb-shards/user-data/init-shard-set.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# GLOBAL VARIABLES +PID_FILE="/run/terraform-shard-manager.pid" +CURRENT_SCRIPT_LOGS="/tmp/bootstrap-cluster.log" + +exec >$CURRENT_SCRIPT_LOGS 2>&1 + +function cleanup() { + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi +} + +function create_pid_file() { + local max_attempts=5 + + for ((i=$max_attempts; i>0;i--)); do + if [ -f "$PID_FILE" ]; then + echo "Script is already running with PID $(cat "$PID_FILE"). Waiting process to end..." + sleep 1m + continue + fi + + echo $$ > "$PID_FILE" + trap cleanup EXIT + return 0 + done + + echo "Exiting due to waiting too long for the process to finish." + exit 1 +} + +function get_replica_shard() { + local public_ips=("$@") + local members="[]" + local config='{"_id": "shard_repl", "members": []}' + + for ((i=0; i<${#public_ips[@]}; i++)) { + local member="{\"_id\": $i, \"host\": \"${public_ips[i]}:28041\"}" + members=$(jq --argjson newMember "$member" '. + [$newMember]' <<<"$members") + } + + config=$(jq --argjson members "$members" '.members += $members' <<<"$config") + + echo "$config" +} + +function get_replica_shard_script() { + local rsconf + local replica_set_config + rsconf=$(get_replica_shard "$@") + replica_set_config=$(cat <