PR from devlop for release v0.0.19

CHANGELOG.rst

@@ -2,6 +2,13 @@
 CHANGELOG
 =========
 
+0.0.19
+======
+* feature:``cli``: Added configure command; easy config setup
+* updates:``docs``: Addtional documentation for configuration options
+* updates:``ami``: Pulled latest CentOS6 errata
+* bugfix:``cfncluster``: Fixed issue with nodewatcher not scaling down
+
 0.0.18
 ======
 * updates:``ami``: Custom CentOS 6 kernel repo added, support for >32 vCPUs

README.rst

@@ -7,3 +7,8 @@ Documentation
 =============
 
 Documentation is part of the project and is published to - http://cfncluster.readthedocs.org/. Of most interest to new users is the Getting Started Guide - http://cfncluster.readthedocs.org/en/latest/getting_started.html.
+
+Issues
+======
+
+Any issues or feedback, should be posted in the AWS HPC forum - https://forums.aws.amazon.com/forum.jspa?forumID=192

amis.txt

@@ -1,10 +1,10 @@
-us-west-2 ami-87de82b7
-eu-central-1 ami-8c97a791
-sa-east-1 ami-2bdd6036
-ap-northeast-1 ami-a8d8c8a9
-eu-west-1 ami-f738be80
-us-east-1 ami-4ad6a522
-us-west-1 ami-298e916c
-ap-southeast-2 ami-efc4aed5
-ap-southeast-1 ami-705a7222
-us-gov-west-1 ami-5fcfa97c
+us-west-2 ami-ab7f599b
+eu-central-1 ami-b43705a9
+sa-east-1 ami-818b359c
+ap-northeast-1 ami-891bfb89
+eu-west-1 ami-87fd69f0
+us-east-1 ami-7660361e
+us-west-1 ami-86b0aac3
+ap-southeast-2 ami-7d730547
+ap-southeast-1 ami-fafcc8a8
+us-gov-west-1 ami-5d68097e

bootstrap/src/scripts/boot_as_compute

@@ -15,6 +15,7 @@ set -x
 
 # Source config
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 # Source functions
 . /opt/cfncluster/scripts/functions.shlib

bootstrap/src/scripts/openlava/boot_as_compute

@@ -14,6 +14,7 @@
 set -x
 
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 function error_exit () {
   script=`basename $0`

bootstrap/src/scripts/sge/boot_as_compute

@@ -14,6 +14,7 @@
 set -x
 
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 # Source functions
 . /opt/cfncluster/scripts/functions.shlib

bootstrap/src/scripts/torque/boot_as_compute

@@ -14,6 +14,7 @@
 set -x
 
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 function error_exit () {
   script=`basename $0`

cli/cfncluster/cfnconfig.py

@@ -14,10 +14,20 @@
 import sys
 import inspect
 import pkg_resources
-import logging
 import json
 import urllib2
 import config_sanity
+import boto.cloudformation
+
+def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack):
+
+    cfn_conn = boto.cloudformation.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+    __stack_name = ('cfncluster-' + stack)
+    __stack = cfn_conn.describe_stacks(stack_name_or_id=__stack_name)[0]
+    __cli_template = [p.value for p in __stack.parameters if p.key == 'CLITemplate'][0]
+
+    return __cli_template
 
 class CfnClusterConfig:
 
@@ -26,6 +36,7 @@ def __init__(self, args):
         self.parameters = []
         self.version = pkg_resources.get_distribution("cfncluster").version
         self.__DEFAULT_CONFIG = False
+        __args_func = self.args.func.func_name
 
         # Determine config file name based on args or default
         if args.config_file is not None:
@@ -50,12 +61,39 @@ def __init__(self, args):
         __config = ConfigParser.ConfigParser()
         __config.read(self.__config_file)
 
+        # Determine the EC2 region to used used or default to us-east-1
+        # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1
+        if args.region:
+            self.region = args.region
+        else:
+            if os.environ.get('AWS_DEFAULT_REGION'):
+                self.region = os.environ.get('AWS_DEFAULT_REGION')
+            else:
+                try:
+                    self.region = __config.get('aws', 'aws_region_name')
+                except ConfigParser.NoOptionError:
+                    self.region = 'us-east-1'
+
+        # Check if credentials have been provided in config
+        try:
+            self.aws_access_key_id = __config.get('aws', 'aws_access_key_id')
+        except ConfigParser.NoOptionError:
+            self.aws_access_key_id=None
+        try:
+            self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key')
+        except ConfigParser.NoOptionError:
+            self.aws_secret_access_key=None
+
         # Determine which cluster template will be used
         try:
             if args.cluster_template is not None:
                 self.__cluster_template = args.cluster_template
             else:
-                self.__cluster_template = __config.get('global', 'cluster_template')
+                if __args_func == 'update':
+                    self.__cluster_template = getStackTemplate(self.region,self.aws_access_key_id,
+                                                               self.aws_secret_access_key, self.args.cluster_name)
+                else:
+                    self.__cluster_template = __config.get('global', 'cluster_template')
         except AttributeError:
             self.__cluster_template = __config.get('global', 'cluster_template')
         self.__cluster_section = ('cluster %s' % self.__cluster_template)
@@ -82,34 +120,11 @@ def __init__(self, args):
             self.__sanity_check = False
         # Only check config on calls that mutate it
         __args_func = self.args.func.func_name
-        if (__args_func == 'create' or __args_func == 'update') and self.__sanity_check is True:
+        if (__args_func == 'create' or __args_func == 'update' or __args_func == 'configure') and self.__sanity_check is True:
             pass
         else:
             self.__sanity_check = False
 
-        # Determine the EC2 region to used used or default to us-east-1
-        # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1
-        if args.region:
-            self.region = args.region
-        else:
-            if os.environ.get('AWS_DEFAULT_REGION'):
-                self.region = os.environ.get('AWS_DEFAULT_REGION')
-            else:
-                try:
-                    self.region = __config.get('aws', 'aws_region_name')
-                except ConfigParser.NoOptionError:
-                    self.region = 'us-east-1'
-
-        # Check if credentials have been provided in config
-        try:
-            self.aws_access_key_id = __config.get('aws', 'aws_access_key_id')
-        except ConfigParser.NoOptionError:
-            self.aws_access_key_id=None
-        try:
-            self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key')
-        except ConfigParser.NoOptionError:
-            self.aws_secret_access_key=None
-
         # Get the EC2 keypair name to be used, exit if not set
         try:
             self.key_name = __config.get(self.__cluster_section, 'key_name')

cli/cfncluster/cli.py

@@ -16,10 +16,14 @@
 import json
 
 import cfncluster
+import easyconfig
 
 def create(args):
     cfncluster.create(args)
 
+def configure(args):
+    easyconfig.configure(args)
+
 def status(args):
     cfncluster.status(args)
 
@@ -121,9 +125,12 @@ def main():
                         help='show the status of cfncluster with the provided name.')
     pinstances.set_defaults(func=instances)
 
+    pconfigure = subparsers.add_parser('configure', help='creating initial cfncluster configuration')
+    pconfigure.set_defaults(func=configure)
+
     pversion = subparsers.add_parser('version', help='display version of cfncluster')
     pversion.set_defaults(func=version)
 
     args = parser.parse_args()
     logging.debug(args)
-    args.func(args)
+    args.func(args)

cli/cfncluster/config_sanity.py

@@ -39,6 +39,13 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty
         except boto.exception.BotoServerError as e:
             print('Config sanity error: %s' % e.message)
             sys.exit(1)
+        # Check for DNS support in the VPC
+        if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsSupport').enable_dns_support:
+            print("DNS Support is not enabled in %s" % test[0].id)
+            sys.exit(1)
+        if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsHostnames').enable_dns_hostnames:
+            print("DNS Hostnames not enabled in %s" % test[0].id)
+            sys.exit(1)
     # VPC Subnet Id
     elif resource_type == 'VPCSubnet':
         try:

cli/cfncluster/easyconfig.py

@@ -0,0 +1,171 @@
+# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the 'License'). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/asl/
+#
+# or in the 'LICENSE.txt' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ConfigParser
+import sys
+import boto.ec2
+import boto.vpc
+import os
+import logging
+
+import cfnconfig
+
+logger = logging.getLogger('cfncluster.cfncluster')
+
+def prompt(prompt, default_value=None, hidden=False, options=None):
+    if hidden and default_value is not None:
+        user_prompt = prompt + ' [*******' + default_value[-4:] + ']: '
+    else:
+        user_prompt = prompt + ' [' 
+        if default_value is not None:
+            user_prompt = user_prompt + default_value + ']: '
+        else:
+            user_prompt = user_prompt + ']: '
+
+    if isinstance(options, list):
+        print 'Acceptable Values for %s: ' % prompt
+        for o in options:
+            print '    %s' % o
+
+    var = raw_input(user_prompt)
+
+    if var == '':
+        return default_value
+    else:
+        return var
+
+def get_regions():
+    regions = boto.ec2.regions()
+    names = []
+    for region in regions:
+        names.append(region.name)
+    return names
+
+def ec2_conn(aws_access_key_id, aws_secret_access_key, aws_region_name):
+    if aws_region_name:
+        region = aws_region_name
+    elif os.environ.get('AWS_DEFAULT_REGION'):
+        region = os.environ.get('AWS_DEFAULT_REGION')
+    else:
+        region = 'us-east-1'
+
+    conn = boto.ec2.connect_to_region(region,aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key)
+    return conn
+
+def vpc_conn(aws_access_key_id, aws_secret_access_key, aws_region_name):
+    if aws_region_name:
+        region = aws_region_name
+    elif os.environ.get('AWS_DEFAULT_REGION'):
+        region = os.environ.get('AWS_DEFAULT_REGION')
+    else:
+        region = 'us-east-1'
+
+    conn = boto.vpc.connect_to_region(region,aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key)
+    return conn
+
+def list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name):
+    conn = ec2_conn(aws_access_key_id, aws_secret_access_key, aws_region_name)
+    keypairs = conn.get_all_key_pairs()
+    keynames = []
+    for key in keypairs:
+        keynames.append(key.name)
+
+    if not keynames:
+        print 'ERROR: No keys found in region ' + aws_region_name
+        print 'Please create an EC2 keypair before continuing'
+        sys.exit(1)
+
+    return keynames
+
+def list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name):
+    conn = vpc_conn(aws_access_key_id, aws_secret_access_key, aws_region_name)
+    vpcs = conn.get_all_vpcs()
+    vpcids = []
+    for vpc in vpcs:
+        vpcids.append(vpc.id)
+
+    if not vpcids:
+        print 'ERROR: No vpcs found in region ' + aws_region_name
+        print 'Please create an EC2 vpcpair before continuing'
+        sys.exit(1)
+
+    return vpcids
+
+def list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_id):
+    conn = vpc_conn(aws_access_key_id, aws_secret_access_key, aws_region_name)
+    subnets = conn.get_all_subnets(filters=[('vpcId', vpc_id)])
+    subnetids = []
+    for subnet in subnets:
+        subnetids.append(subnet.id)
+
+    if not subnetids:
+        print 'ERROR: No subnets found in region ' + aws_region_name
+        print 'Please create an EC2 subnetpair before continuing'
+        sys.exit(1)
+
+    return subnetids
+
+def configure(args):
+
+    # Determine config file name based on args or default
+    if args.config_file is not None:
+        config_file = args.config_file
+    else:
+        config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config'))
+
+    config = ConfigParser.ConfigParser()
+
+    # Check if configuration file exists
+    if os.path.isfile(config_file):
+        config.read(config_file)
+        config_read = True
+
+    # Prompt for required values, using existing as defaults 
+    cluster_template = prompt('Cluster Template', config.get('global', 'cluster_template') if config.has_option('global', 'cluster_template') else 'default')
+    aws_access_key_id = prompt('AWS Access Key ID', config.get('aws', 'aws_access_key_id') if config.has_option('aws', 'aws_access_key_id') else None, True)
+    aws_secret_access_key = prompt('AWS Secret Access Key ID', config.get('aws', 'aws_secret_access_key') if config.has_option('aws', 'aws_secret_access_key') else None, True)
+
+    # Use built in boto regions as an available option
+    aws_region_name = prompt('AWS Region ID', config.get('aws', 'aws_region_name') if config.has_option('aws', 'aws_region_name') else None, options=get_regions())
+    vpcname = prompt('VPC Name', config.get('cluster ' + cluster_template, 'vpc_settings') if config.has_option('cluster ' + cluster_template, 'vpc_settings') else 'public')
+
+    # Query EC2 for available keys as options 
+    key_name = prompt('Key Name', config.get('cluster ' + cluster_template, 'key_name') if config.has_option('cluster ' + cluster_template, 'key_name') else None, options=list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name))
+    vpc_id = prompt('VPC ID', config.get('vpc ' + vpcname, 'vpc_id') if config.has_option('vpc ' + vpcname, 'vpc_id') else None, options=list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name))
+    master_subnet_id = prompt('Master Subnet ID', config.get('vpc ' + vpcname, 'master_subnet_id') if config.has_option('vpc ' + vpcname, 'master_subnet_id') else None, options=list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_id))
+
+    # Dictionary of values we want to set
+    s_global = { '__name__': 'global', 'cluster_template': cluster_template, 'update_check': 'true', 'sanity_check': 'true' }
+    s_aws = { '__name__': 'aws', 'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key, 'aws_region_name': aws_region_name }
+    s_cluster = { '__name__': 'cluster ' + cluster_template, 'key_name': key_name, 'vpc_settings': vpcname }
+    s_vpc = { '__name__': 'vpc ' + vpcname, 'vpc_id': vpc_id, 'master_subnet_id': master_subnet_id }
+
+    sections = [s_aws, s_cluster, s_vpc, s_global]
+
+    # Loop through the configuration sections we care about
+    for section in sections:
+        try:
+            config.add_section(section['__name__'])
+        except ConfigParser.DuplicateSectionError:
+            pass
+        for key, value in section.iteritems():
+            # Only update configuration if not set
+            if value is not None and key is not '__name__':
+                config.set(section['__name__'], key, value)
+
+    # Write configuration to disk
+    cfgfile = open(config_file,'w')
+    config.write(cfgfile)
+    cfgfile.close()
+
+    # Verify the configuration
+    cfnconfig.CfnClusterConfig(args)
+