boxbuilder_remote_build_ami

#!/usr/bin/env bash

#  http://github.com/thewoolleyman/boxbuilder
#  Copyright (c) 2010 Chad Woolley - The MIT License

boxbuilder_debug=${boxbuilder_debug:-false}

if [[ $boxbuilder_debug = true ]]; then
  export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  # set -o verbose # Comment for now, makes debugging too hard, needs to default to false and have flag: http://www.pivotaltracker.com/story/show/5430700
  set -o xtrace
fi

set -o errexit
set -o errtrace
set -o noclobber
set -o nounset
set -o pipefail

_log_prefix="BOXBUILDER - $BASH_SOURCE:"

function onexit() {
  local exit_status=${1:-$?}
  if [[ $exit_status != 0 ]]; then
    _error_line="error trapped."
  else
    _error_line=''
  fi
  if [[ $(type -t onexit_hook) = 'function' ]]; then
    onexit_hook
  fi
  echo "$_log_prefix $_error_line Exiting $0 with $exit_status"
  exit $exit_status
}

function disable_error_checking() {
  trap - ERR
  set +o errexit
}

function enable_error_checking() {
  trap onexit ERR
  set -o errexit
}

trap onexit HUP INT QUIT TERM ERR

exec_remotely() {
  local arg_array i rc
  for arg; do
    arg_array[i++]=$(printf %q "$arg")
  done
  echo "ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host \"$@\""
  ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host "$@"
}

exec_remotely_quiet() {
  local arg_array i rc
  for arg; do
    arg_array[i++]=$(printf %q "$arg")
  done
  # echo "ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host \"$@\""
  ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host "$@"
}

exec_remotely_pseudo_tty() {
  local arg_array i rc
  for arg; do
    arg_array[i++]=$(printf %q "$arg")
  done
  echo "ssh -t -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host \"$@\""
  ssh -t -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host "$@"
}

setup() {
  echo "$_log_prefix  Environment variable setup ..."

  if [ -e $HOME/.boxbuilder_remote_build_amirc ]; then
    source $HOME/.boxbuilder_remote_build_amirc
  fi
  boxbuilder_config=${boxbuilder_config:-'export boxbuilder_config_placeholder_from_boxbuilder_remote_build_ami=boxbuilder_config_placeholder_from_boxbuilder_remote_build_ami'}
  eval $boxbuilder_config
  boxbuilder_branch=${boxbuilder_branch:-"master"}
  boxbuilder_build_ami_url=${boxbuilder_build_ami_url:-"http://github.com/thewoolleyman/boxbuilder/raw/$boxbuilder_branch/boxbuilder_build_ami"}
  boxbuilder_user=${boxbuilder_user:?"Please set 'boxbuilder_user' to the default ssh user defined by the builder instance AMI ('ubuntu' on Ubuntu AMIs)"}
  boxbuilder_builder_instance_id=${boxbuilder_builder_instance_id:-""}
  boxbuilder_builder_instance_instance_type=${boxbuilder_builder_instance_instance_type:-"m1.large"}
  boxbuilder_builder_instance_ami_id=${boxbuilder_builder_instance_ami_id:-"ami-4b4ba522"}
  boxbuilder_ec2_api_tools_url=${boxbuilder_ec2_api_tools_url:-"http://s3.amazonaws.com/ec2-downloads/ec2-api-tools.zip"}
  boxbuilder_terminate_ec2_resources=${boxbuilder_terminate_ec2_resources:-true}
  
  EC2_CERT=${EC2_CERT:-""}
  EC2_PRIVATE_KEY=${EC2_PRIVATE_KEY:-""}
  EC2_KEYPAIR=${EC2_KEYPAIR:-""}

  boxbuilder_skip_build=${boxbuilder_skip_build:-false} # Undocumented flag, for debugging boxbuilder_remote_build_ami

  # Internal constants
  _autoterminating_temp_ec2_resource_tag='boxbuilder_temp_resource_safe_to_terminate'

  # Initialize local variables which may be undefined if boxbuilder_skip_build=true
  _build_ami_output=''
  
  echo "$_log_prefix  Finished environment variable setup:"
  echo "boxbuilder_config=$boxbuilder_config"
  # TODO: Add all env vars which are directly used by this script
}

set_ec2_credentials() {
  echo "$_log_prefix Setting EC2 credentials ..."
  
  if [[ -z $EC2_CERT ]]; then
    EC2_CERT=( $HOME/.ec2/cert-*.pem ) # grab first file using bash array
    echo "$_log_prefix EC2_CERT is not set, loading from $HOME/.ec2/cert-*.pem ..."
  fi

  if [[ ! -e $EC2_CERT ]]; then
    echo "$_log_prefix ERROR: EC2_CERT was not found at $EC2_CERT, exiting ..."
    return 1
  fi

  if [[ -z $EC2_PRIVATE_KEY ]]; then
    EC2_PRIVATE_KEY=( $HOME/.ec2/pk-*.pem ) # grab first file using bash array
    echo "$_log_prefix EC2_PRIVATE_KEY is not set, loading from $HOME/.ec2/pk-*.pem ..."
  fi

  if [[ ! -e $EC2_PRIVATE_KEY ]]; then
    echo "$_log_prefix ERROR: EC2_PRIVATE_KEY was not found at $EC2_PRIVATE_KEY, exiting ..."
    return 1
  fi

  if [[ -z $EC2_KEYPAIR ]]; then
    EC2_KEYPAIR=( $HOME/.ec2/keypair-*.pem ) # grab first file using bash array
    echo "$_log_prefix EC2_KEYPAIR is not set, loading from $HOME/.ec2/keypair-*.pem ..."
  fi

  if [[ ! -e $EC2_KEYPAIR ]]; then
    echo "$_log_prefix ERROR: EC2_KEYPAIR was not found at $EC2_KEYPAIR, exiting ..."
    return 1
  fi

  EC2_KEYPAIR_NAME=${EC2_KEYPAIR_NAME:?"Please set 'EC2_KEYPAIR_NAME' to the name of your EC2 keypair which matches the EC2_KEYPAIR at $EC2_KEYPAIR"}

  export EC2_CERT
  export EC2_PRIVATE_KEY
  export EC2_KEYPAIR

  echo "$_log_prefix Finished setting EC2 credentials.  Using EC2_CERT=$EC2_CERT , EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY , EC2_KEYPAIR=$EC2_KEYPAIR , and EC2_KEYPAIR_NAME=$EC2_KEYPAIR_NAME ..."
}

set_up_ec2_tools() {
  echo "$_log_prefix Setting up EC2 tools ..."
  
  _ec2_tools_path=$HOME/.boxbuilder_ec2_tools
  _ec2_api_tools_path=$_ec2_tools_path/ec2-api-tools
  if [[ -x $_ec2_api_tools_path/bin/ec2-run-instances ]]; then
    echo "$_log_prefix EC2 API Tools already exist at $_ec2_api_tools_path.  Not re-downloading ..."
  else
    mkdir -p $_ec2_tools_path
    if [[ ! $(which unzip) ]]; then
      echo "$_log_prefix ERROR: 'unzip' executable not available.  It is required to unzip auto-downloaded EC2 API Tools. Either put unzip with on your path, or manually unzip the EC2 API Tools to $_ec2_api_tools_path ..."
      return 1
    fi
    _ec2_api_tools_zipfile=$_ec2_tools_path/${boxbuilder_ec2_api_tools_url##*/} # grabs last path component of url to use as zipfile name
    echo "$_log_prefix EC2 API Tools do not exist at $_ec2_api_tools_path.  Downloading from $boxbuilder_ec2_api_tools_url to $_ec2_api_tools_path ..."
    wget -O $_ec2_api_tools_zipfile $boxbuilder_ec2_api_tools_url
    echo "$_log_prefix Reading top-level directory for $_ec2_api_tools_zipfile ..."
    # Zipfile is structured with top-level directory containing the release date, so we must determine what it is, extract, and symlink it to _ec2_api_tools_path
    disable_error_checking
    _first_zipfile_listing_entry=$(unzip -l $_ec2_api_tools_zipfile | head -n4 | tail -n1) # _first_zipfile_listing_entry should be top-level zipfile dir
    enable_error_checking
    _top_level_zipfile_dir=${_first_zipfile_listing_entry##* } # grab just last space-delimited directory component from _first_zipfile_listing_entry
    _top_level_zipfile_dir=${_top_level_zipfile_dir%%/} # strip trailing slash
    echo "$_log_prefix Unzipping $_ec2_api_tools_zipfile to $_ec2_tools_path ..."
    unzip -o -d $_ec2_tools_path $_ec2_api_tools_zipfile # Unzip to $HOME/.boxbuilder_ec2_tools
    echo "$_log_prefix Symlinking $_ec2_tools_path/$_top_level_zipfile_dir to $_ec2_api_tools_path ..."
    ln -sf $_ec2_tools_path/$_top_level_zipfile_dir $_ec2_api_tools_path # symlink to simple directory name without release date.
  fi
  export EC2_HOME=$_ec2_api_tools_path
  export PATH=$EC2_HOME/bin:$PATH
  echo "$_log_prefix Finished setting up EC2 tools ..."
}

start_builder_instance() {
  echo "$_log_prefix Starting builder instance to perform AMI build: ami=$boxbuilder_builder_instance_ami_id, instance_type=$boxbuilder_builder_instance_instance_type ..."
  local boxbuilder_ec2_resource_tag=${boxbuilder_ec2_resource_tag:-$_autoterminating_temp_ec2_resource_tag}
  boxbuilder_builder_instance_id=$(ec2-run-instances --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT \
    --key "$EC2_KEYPAIR_NAME" \
    --availability-zone us-east-1a \
    --instance-type "$boxbuilder_builder_instance_instance_type" \
    "$boxbuilder_builder_instance_ami_id" \
    -d "$boxbuilder_ec2_resource_tag" \
    | egrep ^INSTANCE | cut -f2)
  echo "$_log_prefix  Started EC2 builder instance: instance_id=$boxbuilder_builder_instance_id"
}

get_builder_instance_host() {
  echo "$_log_prefix Finding hostname for $boxbuilder_builder_instance_id ..."
  _instance_startup_elapsed_time=${_instance_startup_elapsed_time:-}
  let _instance_startup_elapsed_time=1 # HACK: don't initialize to zero avoid error due to nonzero rc.  arithmetic false is not shell false.  zero is false is one :)  See http://wiki.bash-hackers.org/syntax/arith_expr#arithmetic_expressions_and_return_codes
  let _instance_startup_check_interval=10
  while boxbuilder_builder_instance_host=$(ec2-describe-instances --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT \
     "$boxbuilder_builder_instance_id" | \
    egrep ^INSTANCE | cut -f4) && [[ -z $boxbuilder_builder_instance_host ]]
    do
      echo -n .
      sleep $_instance_startup_check_interval
      let _instance_startup_elapsed_time=$_instance_startup_elapsed_time+$_instance_startup_check_interval
      if (( $_instance_startup_elapsed_time > 150 )); then
        echo "$_log_prefix Unable to start instance for AMI $boxbuilder_builder_instance_ami_id after $_instance_startup_elapsed_time seconds.  Aborting."
        return 1
      fi
  done
  echo
  echo "$_log_prefix Got hostname $boxbuilder_builder_instance_host for instance $boxbuilder_builder_instance_id"
}

verify_builder_instance_host_ssh_connectivity() {
  echo "$_log_prefix Waiting for ssh connectivity to $boxbuilder_builder_instance_host for either $boxbuilder_user or root..."
  let _instance_startup_elapsed_time=1
  let _instance_startup_check_interval=10
  while
    boxbuilder_user_ssh_up=$(ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR $boxbuilder_user@$boxbuilder_builder_instance_host "echo 'ssh up'")
    root_ssh_up=$(ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR root@$boxbuilder_builder_instance_host "echo 'ssh up'")
    [[ "$boxbuilder_user_ssh_up" != "ssh up" && "$root_ssh_up" != "ssh up" ]]
    do
      echo -n .
      sleep $_instance_startup_check_interval
      let _instance_startup_elapsed_time=$_instance_startup_elapsed_time+$_instance_startup_check_interval
      echo "$_log_prefix SSH connect response from $boxbuilder_user@$boxbuilder_builder_instance_host: $boxbuilder_user_ssh_up"
      echo "$_log_prefix SSH connect response from root@$boxbuilder_builder_instance_host: $root_ssh_up"
      echo "$_log_prefix Trying to SSH to $boxbuilder_builder_instance_host as $boxbuilder_user or root again after 5 seconds..."
      if (( $_instance_startup_elapsed_time > 150 )); then
          echo "$_log_prefix Unable to SSH to $boxbuilder_builder_instance_host as $boxbuilder_user or root after $_instance_startup_elapsed_time seconds.  Aborting."
        return 1
      fi
  done
  if [[ "$root_ssh_up" = "ssh up" ]]; then
    _root_ssh_access_exists=true
  else
    _root_ssh_access_exists=false
  fi
  echo
}

exec_remotely_quiet_root() {
  local arg_array i rc
  for arg; do
    arg_array[i++]=$(printf %q "$arg")
  done
  ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR root@$boxbuilder_builder_instance_host "$@"
}

grant_nopasswd_sudo_access_from_boxbuilder_user() {
  echo "$_log_prefix  Checking 'admin' group membership for user $boxbuilder_user on host $boxbuilder_builder_instance_host."
  if ! exec_remotely_quiet "groups | grep -q -e 'admin'"; then
    echo "$_log_prefix error on line $LINENO - user $boxbuilder_user is not a member of the 'admin' group on host $boxbuilder_builder_instance_host.  Please add..."
    return 1
  fi

  echo "$_log_prefix  Checking no-password and no requiretty sudo access for 'admin' group on host $boxbuilder_builder_instance_host and adding if necessary.  If prompted, please type your password, or ctrl-c to abort..."

  _needs_nopasswd_sudoers_added=false
  if ! exec_remotely_quiet "sudo grep -e '%admin.*ALL.*=.*NOPASSWD.*:.*ALL' /etc/sudoers"; then
    _needs_nopasswd_sudoers_added=true
  fi

  _needs_requiretty_sudoers_removed=false
  if exec_remotely_quiet "sudo grep 'requiretty' /etc/sudoers"; then
    _needs_requiretty_sudoers_removed=true
  fi

  if ! $_needs_nopasswd_sudoers_added && ! $_needs_requiretty_sudoers_removed; then
    echo "$_log_prefix  no sudoers changes needed ..."
    return 0
  fi

  exec_remotely_quiet "sudo cp /etc/sudoers /etc/sudoers.bak"

  _sudoers_additional_error_instructions='Please use visudo to manually make these changes: '
  if $_needs_requiretty_sudoers_removed; then
    exec_remotely_quiet "sudo cat /etc/sudoers | grep -v 'requiretty' > /tmp/sudoers.new"
    _sudoers_additional_error_instructions="$_sudoers_additional_error_instructions Remove 'requiretty' line. "
  else
    exec_remotely_quiet "sudo cp /etc/sudoers /tmp/sudoers.new"
  fi
  exec_remotely_quiet "sudo chmod 0440 /tmp/sudoers.new"

  if $_needs_nopasswd_sudoers_added; then
    echo "$_log_prefix  Giving user $boxbuilder_user NO PASSWORD sudo privileges"
    _new_sudoers_line="%admin ALL=NOPASSWD: ALL"
    exec_remotely_quiet "sudo sh -c 'echo \"$_new_sudoers_line\" >> /tmp/sudoers.new' "
    _sudoers_additional_error_instructions="$_sudoers_additional_error_instructions Add a line '$_new_sudoers_line'. "
  fi

  if ! exec_remotely_quiet "test -e /tmp/sudoers.new"; then
    echo "$_log_prefix error on line $LINENO - Could not add create /tmp/sudoers.new on host $boxbuilder_builder_instance_host.  $_sudoers_additional_error_instructions"
    return 1
  fi

  if ! exec_remotely_quiet "sudo visudo -c -s -f /tmp/sudoers.new"; then
    echo "$_log_prefix error on line $LINENO -: Syntax error in /tmp/sudoers.new on host $boxbuilder_builder_instance_host.  $_sudoers_additional_error_instructions"
    return 1
  fi
  exec_remotely_quiet "sudo cp /tmp/sudoers.new /etc/sudoers"
}

grant_nopasswd_sudo_access_from_root() {
  echo "$_log_prefix  Checking no-password and no requiretty sudo access for 'admin' group on host $boxbuilder_builder_instance_host and adding if necessary.  If prompted, please type your password, or ctrl-c to abort..."

  _needs_nopasswd_sudoers_added=false
  if ! exec_remotely_quiet_root "grep -e '%admin.*ALL.*=.*NOPASSWD.*:.*ALL' /etc/sudoers"; then
    _needs_nopasswd_sudoers_added=true
  fi

  _needs_requiretty_sudoers_removed=false
  if exec_remotely_quiet_root "grep 'requiretty' /etc/sudoers"; then
    _needs_requiretty_sudoers_removed=true
  fi

  if ! $_needs_nopasswd_sudoers_added && ! $_needs_requiretty_sudoers_removed; then
    echo "$_log_prefix  no sudoers changes needed ..."
    return 0
  fi

  exec_remotely_quiet_root "cp /etc/sudoers /etc/sudoers.bak"

  _sudoers_additional_error_instructions='Please use visudo to manually make these changes: '
  if $_needs_requiretty_sudoers_removed; then
    exec_remotely_quiet_root "cat /etc/sudoers | grep -v 'requiretty' > /tmp/sudoers.new"
    _sudoers_additional_error_instructions="$_sudoers_additional_error_instructions Remove 'requiretty' line. "
  else
    exec_remotely_quiet_root "cp /etc/sudoers /tmp/sudoers.new"
  fi
  exec_remotely_quiet_root "chmod 0440 /tmp/sudoers.new"

  if $_needs_nopasswd_sudoers_added; then
    echo "$_log_prefix  Giving user $boxbuilder_user NO PASSWORD sudo privileges"
    _new_sudoers_line="%admin ALL=NOPASSWD: ALL"
    exec_remotely_quiet_root "sh -c 'echo \"$_new_sudoers_line\" >> /tmp/sudoers.new' "
    _sudoers_additional_error_instructions="$_sudoers_additional_error_instructions Add a line '$_new_sudoers_line'. "
  fi

  if ! exec_remotely_quiet_root "test -e /tmp/sudoers.new"; then
    echo "$_log_prefix error on line $LINENO - Could not add create /tmp/sudoers.new on host $boxbuilder_builder_instance_host.  $_sudoers_additional_error_instructions"
    return 1
  fi

  if ! exec_remotely_quiet_root "visudo -c -s -f /tmp/sudoers.new"; then
    echo "$_log_prefix error on line $LINENO -: Syntax error in /tmp/sudoers.new on host $boxbuilder_builder_instance_host.  $_sudoers_additional_error_instructions"
    return 1
  fi
  exec_remotely_quiet_root "cp /tmp/sudoers.new /etc/sudoers"
}

grant_nopasswd_sudo_access() {
  if $_root_ssh_access_exists; then
    # attempt to install sudo if it isn't already installed
    exec_remotely_quiet_root "yum install -y sudo"
    grant_nopasswd_sudo_access_from_root
  else
    grant_nopasswd_sudo_access_from_boxbuilder_user
  fi
}

ensure_boxbuilder_user_exists() {
  echo "$_log_prefix  Ensuring boxbuilder_user '$boxbuilder_user' exists on $boxbuilder_builder_instance_host ..."
  if [[ $boxbuilder_user = 'root' ]]; then
    echo "$_log_prefix  boxbuilder_user cannot be root.  Please set \
boxbuilder_user to a non-root user.  If it does not exist, it will be \
automatically created if there is a valid no-passphrase SSH key to the root user ..."
    return 1
  fi
  if exec_remotely 'id'; then
    echo "$_log_prefix  boxbuilder_user '$boxbuilder_user' already exists on $boxbuilder_builder_instance_host ..."
    # TODO: should verify admin group exists before returning
    return 0
  fi
  if $_root_ssh_access_exists; then
    echo "$_log_prefix  boxbuilder_user '$boxbuilder_user' does not exist, but root SSH access exists on $boxbuilder_builder_instance_host to automatically create boxbuilder_user $boxbuilder_user ..."
  else
    echo "$_log_prefix  boxbuilder_user '$boxbuilder_user' does not exist, and no root SSH access on $boxbuilder_builder_instance_host; cannot automatically create boxbuilder_user $boxbuilder_user.  Create manually with valid SSH access or give SSH access to root ..."
    return 1
  fi
  echo "$_log_prefix  Ensuring 'admin' group exists ..."
  exec_remotely_quiet_root "groupadd -f admin"
  _admin_gid=$(exec_remotely_quiet_root "grep 'admin' /etc/group | cut -d: -f3")
  echo "$_log_prefix  Automatically creating '$boxbuilder_user' user ..."
  if exec_remotely_quiet_root "id $boxbuilder_user"; then
    echo "$_log_prefix  '$boxbuilder_user' user already exists, not re-adding ..."
  else
    exec_remotely_quiet_root "useradd $boxbuilder_user -m -g $_admin_gid"
  fi
  echo "$_log_prefix  Finding home dir for '$boxbuilder_user' user ..."
  _boxbuilder_user_home=$(exec_remotely_quiet_root "su - $boxbuilder_user -c 'echo \$HOME'")
  echo "$_log_prefix  Copying SSH authorized keys from root to '$boxbuilder_user' user's home dir with correct ownership ..."
  exec_remotely_quiet_root "mkdir -p '$_boxbuilder_user_home/.ssh/'"
  exec_remotely_quiet_root "cp '.ssh/authorized_keys' '$_boxbuilder_user_home/.ssh/'"
  exec_remotely_quiet_root "chown -R $boxbuilder_user '$_boxbuilder_user_home/.ssh'"
  if exec_remotely 'id'; then
    echo "$_log_prefix  boxbuilder_user '$boxbuilder_user' successfully created on $boxbuilder_builder_instance_host ..."
  else
    echo "$_log_prefix  boxbuilder_user '$boxbuilder_user' was NOT successfully created on $boxbuilder_builder_instance_host - something went wrong :( - try setting boxbuilder_debug=true ..."
    return 1
  fi
}

upload_ec2_credentials() {
  echo "$_log_prefix Uploading EC2 credentials to $boxbuilder_builder_instance_host ..."
  exec_remotely 'mkdir -p $HOME/.ec2'
  echo "$_log_prefix Uploading EC2_CERT $EC2_CERT to $boxbuilder_user@$boxbuilder_builder_instance_host:.ec2/ ..."
  rsync \
    --rsh="ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR" \
    --rsync-path="rsync" \
    $EC2_CERT \
    $boxbuilder_user@$boxbuilder_builder_instance_host:.ec2/

  echo "$_log_prefix Uploading EC2_PRIVATE_KEY $EC2_PRIVATE_KEY to $boxbuilder_user@$boxbuilder_builder_instance_host:.ec2/ ..."
  rsync \
    --rsh="ssh -o 'StrictHostKeyChecking no' -o 'PasswordAuthentication no' -i $EC2_KEYPAIR" \
    --rsync-path="rsync" \
    $EC2_PRIVATE_KEY \
    $boxbuilder_user@$boxbuilder_builder_instance_host:.ec2/

  echo "$_log_prefix Finished uploading EC2 credentials to $boxbuilder_builder_instance_host ..."
}

remote_build_ami() {
  _boxbuilder_build_ami_script=/tmp/${boxbuilder_build_ami_url##*/} # grabs last path component of url to use as script name
  echo "$_log_prefix Downloading $boxbuilder_build_ami_url to $_boxbuilder_build_ami_script on $boxbuilder_builder_instance_host ..."
  exec_remotely "wget -O $_boxbuilder_build_ami_script $boxbuilder_build_ami_url"
  while exists=$(exec_remotely_quiet "if [[ -e $_boxbuilder_build_ami_script ]] ; then echo exists; fi"); [[ "$exists" != "exists" ]]; do
    echo "$_log_prefix Waiting for wget to finish downloading  $boxbuilder_build_ami_url to $_boxbuilder_build_ami_script on $boxbuilder_builder_instance_host ..."
    sleep 5
  done
  echo "$_log_prefix $boxbuilder_build_ami_url successfully downloaded to $_boxbuilder_build_ami_script on $boxbuilder_builder_instance_host ..."

  echo "$_log_prefix Changing permissions on $_boxbuilder_build_ami_script on $boxbuilder_builder_instance_host ..."
  exec_remotely "chmod a+x $_boxbuilder_build_ami_script"
  echo "$_log_prefix Running $_boxbuilder_build_ami_script on $boxbuilder_builder_instance_host ..."
  _build_ami_output=$(exec_remotely_pseudo_tty "export boxbuilder_config='$boxbuilder_config' \
    && export boxbuilder_user=$boxbuilder_user \
    && export boxbuilder_debug=$boxbuilder_debug \
    && $_boxbuilder_build_ami_script")
  echo "$_build_ami_output"
}

boxbuilder_terminate_builder_instances() {
  if [[ $boxbuilder_debug = true ]]; then
    echo "$_log_prefix Terminating EC2 builder instances ..."
  fi
  _instances=$(ec2-describe-instances --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT)
  disable_error_checking # we must disable error checking because grep will fail with nonzero if there are no running instances
  _instances=$(echo "$_instances" | egrep ^INSTANCE)
  enable_error_checking
  echo "$_instances" | while read line; do
    if [[ -z $line ]]; then
      # ignore any blank lines that got through previous grepping
      continue
    fi
    _instance_id=$(echo "$line" | cut -f2)
    if [[ $boxbuilder_debug = true ]]; then
      echo "$_log_prefix Checking instance id $_instance_id for User Data containing '$_autoterminating_temp_ec2_resource_tag'"
    fi

    # Do not terminate instances which boxbuilder did not start (which do not contain $_autoterminating_temp_ec2_resource_tag)
    _user_data=$(ec2-describe-instance-attribute --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT \
      $_instance_id --user-data \
      | cut -f3)
    if [[ $boxbuilder_debug = true ]]; then
      echo "$_log_prefix Instance id $_instance_id User Data: $_user_data"
    fi

    disable_error_checking
    if [[ -z $(echo "$_user_data" | grep $_autoterminating_temp_ec2_resource_tag) ]]; then
      if [[ $boxbuilder_debug = true ]]; then
        echo "$_log_prefix NOT terminating instance $_instance_id because it does not contain '$_autoterminating_temp_ec2_resource_tag' in the User Data"
      fi
      continue
    fi
    enable_error_checking

    disable_error_checking
    # Do not attempt to terminate instances which are already terminated
    if [[ -e $(echo "$_user_data" | egrep 'terminated$') ]]; then
      if [[ $boxbuilder_debug = true ]]; then
        echo "$_log_prefix NOT terminating instance $_instance_id because it is already terminated"
      fi
      continue
    fi
    enable_error_checking

    echo "$_log_prefix terminating boxbuilder builder instance $_instance_id ..."
    disable_error_checking
    _terminate_instances_result=$(ec2-terminate-instances --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT "$_instance_id")
    enable_error_checking
    echo "$_log_prefix ec2-terminate-instances result for instance $_instance_id:"
    echo "$(sed -n l <<< $_terminate_instances_result)"
  done
  if [[ $boxbuilder_debug = true ]]; then
    echo "$_log_prefix Finished terminating EC2 builder instances ..."
  fi
}

boxbuilder_deregister_temp_amis() {
  if [[ $boxbuilder_debug = true ]]; then
    echo "$_log_prefix Deregistering temporary AMIs ..."
  fi
  _images="$(ec2-describe-images --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT)"
  disable_error_checking # we must disable error checking because grep will fail with nonzero if there are no temp AMIs
  _images="$(echo "$_images" | grep $_autoterminating_temp_ec2_resource_tag)"

  enable_error_checking
  echo "$_images" | while read line; do
    if [[ -z $line ]]; then
      # ignore any blank lines that got through previous grepping
      continue
    fi
    _image_id=$(echo "$line" | cut -f2)

    set +o xtrace # even if boxbuilder_debug is on, always turn off tracing
                  # while working with $_build_ami_output 
                  # dumping entire build output makes test output huge and confusing
    if [[ -n $_build_ami_output ]] ; then
      disable_error_checking
      _created_ami_id_line=$(echo "$_build_ami_output" | grep -e '^boxbuilder_created_ami_id')
      if [[ $boxbuilder_debug = true ]]; then
        # turn back on tracing if we are debugging
        set -o xtrace
      fi
      
      # Do not attempt to deregister temp AMI which was just built
      if [[ -n $(echo "$_created_ami_id_line" | grep $_image_id) ]]; then
        if [[ $boxbuilder_debug = true ]]; then
          echo "$_log_prefix NOT deregistering $_image_id because it is was just created in this boxbuilder_remote_build_ami run"
        fi
        continue
      fi
      enable_error_checking
    fi
    if [[ $boxbuilder_debug = true ]]; then
      # turn back on tracing if we are debugging
      set -o xtrace
    fi

    echo "$_log_prefix deregistering boxbuilder temp AMI $_image_id ..."
    disable_error_checking
    _deregister_result=$(ec2-deregister --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT "$_image_id")
    enable_error_checking
    echo "$_log_prefix ec2-deregister result for image $_image_id:"
    echo "$(sed -n l <<< $_deregister_result)"
  done
  if [[ $boxbuilder_debug = true ]]; then
    echo "$_log_prefix Finished deregistering temporary AMIs ..."
  fi
}

boxbuilder_delete_ebs_snapshots() {
  if [[ $boxbuilder_debug = true ]]; then
    echo "$_log_prefix Deleting EBS snapshots ..."
  fi
  # Do not terminate snapshots which boxbuilder did not start (which do not contain $_autoterminating_temp_ec2_resource_tag)
  _snapshots=$(ec2-describe-snapshots --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT)
  disable_error_checking # we must disable error checking because grep will fail with nonzero if there are no snapshots
  _snapshots=$(echo "$_snapshots" | grep $_autoterminating_temp_ec2_resource_tag)
  enable_error_checking
  echo "$_snapshots" | while read line; do
    if [[ -z $line ]]; then
      # ignore any blank lines that got through previous grepping
      continue
    fi
    _snapshot_id=$(echo "$line" | cut -f2)
    if [[ $boxbuilder_debug = true ]]; then
      echo "$_log_prefix Checking if $_snapshot_id is currently in use by any AMIs ..."
    fi

    # Do not attempt to delete snapshots which are currently in use by an AMI, it isn't allowed
    disable_error_checking
    _image_using_snapshot=$(ec2-describe-images --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT | grep $_snapshot_id )
    enable_error_checking
    if [[ -n $_image_using_snapshot ]]; then
      if [[ $boxbuilder_debug = true ]]; then
        echo "$_log_prefix NOT deleting snapshot ID $_snapshot_id because it is in use by AMI ID $_image_using_snapshot"
      fi
      continue
    fi

    echo "$_log_prefix terminating boxbuilder AMI snapshot $_snapshot_id ..."
    disable_error_checking
    _delete_snapshot_result=$(ec2-delete-snapshot --private-key $EC2_PRIVATE_KEY --cert $EC2_CERT "$_snapshot_id")
    enable_error_checking
    echo "$_log_prefix ec2-delete-snapshot result for snapshot $_snapshot_id:"
    echo "$(sed -n l <<< $_delete_snapshot_result)"
  done
  if [[ $boxbuilder_debug = true ]]; then
    echo "$_log_prefix Finished deleting EBS snapshots ..."
  fi
}

boxbuilder_terminate_ec2_resources() {
  if [[ $boxbuilder_terminate_ec2_resources = true ]] ; then
    echo "$_log_prefix Terminating EC2 resources, because boxbuilder_terminate_ec2_resources was equal to true ..."
    boxbuilder_terminate_builder_instances
    boxbuilder_deregister_temp_amis
    boxbuilder_delete_ebs_snapshots
    echo "$_log_prefix Finished terminating EC2 resources ..."
  else
    echo "$_log_prefix boxbuilder_terminate_ec2_resources was not equal to true, so NO boxbuilder resources were terminated."
  fi
}

run() {
  echo "$_log_prefix Starting ..."
  setup
  set_ec2_credentials
  set_up_ec2_tools
  if [[ $boxbuilder_skip_build = false ]] ; then
    if [[ -z $boxbuilder_builder_instance_id ]] ; then
      start_builder_instance
    fi
    get_builder_instance_host
    verify_builder_instance_host_ssh_connectivity
    ensure_boxbuilder_user_exists
    grant_nopasswd_sudo_access
    upload_ec2_credentials
    remote_build_ami
  fi
  boxbuilder_terminate_ec2_resources
  echo "$_log_prefix Finished and exiting successfully"
}

run
onexit