From 64c4c585b417d32840536346b0914c6f0c074f30 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Mon, 26 Jan 2015 09:27:13 -0800 Subject: [PATCH 01/13] Added DNS Hostname support to VPC Sanity Check --- cli/cfncluster/config_sanity.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index e093bf22e7..376b944af3 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -39,6 +39,12 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty except boto.exception.BotoServerError as e: print('Config sanity error: %s' % e.message) sys.exit(1) + + # Check for DNS support in the VPC + if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsSupport').enable_dns_support: + raise Exception("DNS Support is not enabled in %s" % test[0].id) + if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsHostnames').enable_dns_hostnames: + raise Exception("DNS Hostnames not enabled in %s" % test[0].id) # VPC Subnet Id elif resource_type == 'VPCSubnet': try: From b55f3b8a393ca044b8e758ca3136e43cce5ff0e3 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Mon, 26 Jan 2015 09:52:56 -0800 Subject: [PATCH 02/13] Updating style for exception --- cli/cfncluster/config_sanity.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index 376b944af3..71efeb5fc1 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -39,12 +39,13 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty except boto.exception.BotoServerError as e: print('Config sanity error: %s' % e.message) sys.exit(1) - # Check for DNS support in the VPC if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsSupport').enable_dns_support: - raise Exception("DNS Support is not enabled in %s" % test[0].id) + print("DNS Support is not enabled in %s" % test[0].id) + sys.exit(1) if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsHostnames').enable_dns_hostnames: - raise Exception("DNS Hostnames not enabled in %s" % test[0].id) + print("DNS Hostnames not enabled in %s" % test[0].id) + sys.exit(1) # VPC Subnet Id elif resource_type == 'VPCSubnet': try: From de77a19034bb009fdae71d4885c24087177604e7 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Fri, 13 Feb 2015 22:15:27 -0700 Subject: [PATCH 03/13] Update README.rst --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 168269bd60..fa9a703628 100644 --- a/README.rst +++ b/README.rst @@ -7,3 +7,8 @@ Documentation ============= Documentation is part of the project and is published to - http://cfncluster.readthedocs.org/. Of most interest to new users is the Getting Started Guide - http://cfncluster.readthedocs.org/en/latest/getting_started.html. + +Issues +====== + +Any issues or feedback, should be posted in the AWS HPC forum - https://forums.aws.amazon.com/forum.jspa?forumID=192 From f75b4ba2c053827da6f831db262c9358297bef75 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Mon, 26 Jan 2015 22:11:06 -0800 Subject: [PATCH 04/13] Removing need for --cluster-template parameter when updating a cluster --- cli/cfncluster/cfnconfig.py | 66 ++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index 20e50f9994..3148e55291 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -14,10 +14,20 @@ import sys import inspect import pkg_resources -import logging import json import urllib2 import config_sanity +import boto.cloudformation + +def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): + + cfn_conn = boto.cloudformation.connect_to_region(region,aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key) + __stack_name = ('cfncluster-' + stack) + __stack = cfn_conn.describe_stacks(stack_name_or_id=__stack_name)[0] + __cli_template = [p.value for p in __stack.parameters if p.key == 'CLITemplate'][0] + + return __cli_template class CfnClusterConfig: @@ -26,6 +36,7 @@ def __init__(self, args): self.parameters = [] self.version = pkg_resources.get_distribution("cfncluster").version self.__DEFAULT_CONFIG = False + __args_func = self.args.func.func_name # Determine config file name based on args or default if args.config_file is not None: @@ -50,12 +61,39 @@ def __init__(self, args): __config = ConfigParser.ConfigParser() __config.read(self.__config_file) + # Determine the EC2 region to used used or default to us-east-1 + # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1 + if args.region: + self.region = args.region + else: + if os.environ.get('AWS_DEFAULT_REGION'): + self.region = os.environ.get('AWS_DEFAULT_REGION') + else: + try: + self.region = __config.get('aws', 'aws_region_name') + except ConfigParser.NoOptionError: + self.region = 'us-east-1' + + # Check if credentials have been provided in config + try: + self.aws_access_key_id = __config.get('aws', 'aws_access_key_id') + except ConfigParser.NoOptionError: + self.aws_access_key_id=None + try: + self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key') + except ConfigParser.NoOptionError: + self.aws_secret_access_key=None + # Determine which cluster template will be used try: if args.cluster_template is not None: self.__cluster_template = args.cluster_template else: - self.__cluster_template = __config.get('global', 'cluster_template') + if __args_func == 'update': + self.__cluster_template = getStackTemplate(self.region,self.aws_access_key_id, + self.aws_secret_access_key, self.args.cluster_name) + else: + self.__cluster_template = __config.get('global', 'cluster_template') except AttributeError: self.__cluster_template = __config.get('global', 'cluster_template') self.__cluster_section = ('cluster %s' % self.__cluster_template) @@ -81,35 +119,11 @@ def __init__(self, args): except ConfigParser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it - __args_func = self.args.func.func_name if (__args_func == 'create' or __args_func == 'update') and self.__sanity_check is True: pass else: self.__sanity_check = False - # Determine the EC2 region to used used or default to us-east-1 - # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1 - if args.region: - self.region = args.region - else: - if os.environ.get('AWS_DEFAULT_REGION'): - self.region = os.environ.get('AWS_DEFAULT_REGION') - else: - try: - self.region = __config.get('aws', 'aws_region_name') - except ConfigParser.NoOptionError: - self.region = 'us-east-1' - - # Check if credentials have been provided in config - try: - self.aws_access_key_id = __config.get('aws', 'aws_access_key_id') - except ConfigParser.NoOptionError: - self.aws_access_key_id=None - try: - self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key') - except ConfigParser.NoOptionError: - self.aws_secret_access_key=None - # Get the EC2 keypair name to be used, exit if not set try: self.key_name = __config.get(self.__cluster_section, 'key_name') From c1be23c9de75da5da4871fea6d8ef9b46912287f Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Mon, 26 Jan 2015 22:11:26 -0800 Subject: [PATCH 05/13] Just come cleanup --- cloudformation/cfncluster.cfn.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 120d8c940a..17a3519945 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -7,7 +7,7 @@ "Type" : "AWS::EC2::KeyPair::KeyName" }, "MasterInstanceType" : { - "Description" : "Master Server EC2 instance type", + "Description" : "MasterServer EC2 instance type", "Type" : "String", "Default" : "t2.micro", "ConstraintDescription" : "must be a valid EC2 instance type.", @@ -46,7 +46,7 @@ ] }, "ComputeInstanceType" : { - "Description" : "Cluster Server EC2 instance type", + "Description" : "ComputeFleet EC2 instance type", "Type" : "String", "Default" : "t2.micro", "ConstraintDescription" : "must be a valid EC2 instance type.", From b020b9d6a6e2249e0a1795f08f92cbc8a3349141 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Wed, 18 Feb 2015 10:34:31 -0800 Subject: [PATCH 06/13] Fixing bug with non-global variables when called selfTerminate --- node/nodewatcher/nodewatcher.py | 4 ++-- node/setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node/nodewatcher/nodewatcher.py b/node/nodewatcher/nodewatcher.py index ea5b584a24..fae4c64876 100755 --- a/node/nodewatcher/nodewatcher.py +++ b/node/nodewatcher/nodewatcher.py @@ -97,7 +97,7 @@ def getJobs(s,hostname): return _jobs -def selfTerminate(asg): +def selfTerminate(region,asg,instance_id): _as_conn = boto.ec2.autoscale.connect_to_region(region,proxy=boto.config.get('Boto', 'proxy'), proxy_port=boto.config.get('Boto', 'proxy_port')) _asg = _as_conn.get_all_groups(names=[asg])[0] @@ -126,7 +126,7 @@ def main(): print('Percent of hour used: %d' % hour_percentile) if hour_percentile > 95: - selfTerminate(asg) + selfTerminate(region,asg,instance_id) if __name__ == "__main__": main() diff --git a/node/setup.py b/node/setup.py index 568dd750f9..a7094c09d9 100644 --- a/node/setup.py +++ b/node/setup.py @@ -21,7 +21,7 @@ def read(fname): console_scripts = ['sqswatcher = sqswatcher.sqswatcher:main', 'nodewatcher = nodewatcher.nodewatcher:main'] -version = "0.0.2" +version = "0.0.3" requires = ['boto>=2.34', 'paramiko', 'python-dateutil'] if sys.version_info[:2] == (2, 6): From 9da419c934eb453116c981567a70b43c2454e44d Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Wed, 18 Feb 2015 10:40:34 -0800 Subject: [PATCH 07/13] Ensuring cfn_master is always a short hostname --- bootstrap/src/scripts/boot_as_compute | 1 + bootstrap/src/scripts/openlava/boot_as_compute | 1 + bootstrap/src/scripts/sge/boot_as_compute | 1 + bootstrap/src/scripts/torque/boot_as_compute | 1 + 4 files changed, 4 insertions(+) diff --git a/bootstrap/src/scripts/boot_as_compute b/bootstrap/src/scripts/boot_as_compute index 1b92084983..a6cf62f0cf 100755 --- a/bootstrap/src/scripts/boot_as_compute +++ b/bootstrap/src/scripts/boot_as_compute @@ -15,6 +15,7 @@ set -x # Source config . /opt/cfncluster/cfnconfig +cfn_master=$(echo $cfn_master|cut -d. -f1) # Source functions . /opt/cfncluster/scripts/functions.shlib diff --git a/bootstrap/src/scripts/openlava/boot_as_compute b/bootstrap/src/scripts/openlava/boot_as_compute index e625322675..fdc569ec7a 100755 --- a/bootstrap/src/scripts/openlava/boot_as_compute +++ b/bootstrap/src/scripts/openlava/boot_as_compute @@ -14,6 +14,7 @@ set -x . /opt/cfncluster/cfnconfig +cfn_master=$(echo $cfn_master|cut -d. -f1) function error_exit () { script=`basename $0` diff --git a/bootstrap/src/scripts/sge/boot_as_compute b/bootstrap/src/scripts/sge/boot_as_compute index 7611596a59..90a6d704f6 100755 --- a/bootstrap/src/scripts/sge/boot_as_compute +++ b/bootstrap/src/scripts/sge/boot_as_compute @@ -14,6 +14,7 @@ set -x . /opt/cfncluster/cfnconfig +cfn_master=$(echo $cfn_master|cut -d. -f1) # Source functions . /opt/cfncluster/scripts/functions.shlib diff --git a/bootstrap/src/scripts/torque/boot_as_compute b/bootstrap/src/scripts/torque/boot_as_compute index 19cc5d3db2..e9d5978118 100755 --- a/bootstrap/src/scripts/torque/boot_as_compute +++ b/bootstrap/src/scripts/torque/boot_as_compute @@ -14,6 +14,7 @@ set -x . /opt/cfncluster/cfnconfig +cfn_master=$(echo $cfn_master|cut -d. -f1) function error_exit () { script=`basename $0` From c2c46d0aa94ef67ae3098fcbc5aff3c803e14b41 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Wed, 18 Feb 2015 10:41:12 -0800 Subject: [PATCH 08/13] Bump version number --- docs/source/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 6b476c7943..0cfc2ce06d 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -9,7 +9,7 @@ cfncluster is a framework that deploys and maintains HPC clusters on AWS. It is Installing cfncluster ===================== -The current working version is cfncluster-0.0.17. The CLI is written in python and uses BOTO for AWS actions. You can install the CLI with the following commands, depending on your OS. +The current working version is cfncluster-0.0.19. The CLI is written in python and uses BOTO for AWS actions. You can install the CLI with the following commands, depending on your OS. Linux/OSX --------- From 40832287e05d8007ceaaf4705734cac634d8e6d5 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Thu, 5 Feb 2015 14:57:18 -0800 Subject: [PATCH 09/13] First pass at configuration wizard. --- cli/cfncluster/cfnconfig.py | 3 +- cli/cfncluster/cli.py | 11 +- cli/cfncluster/easyconfig.py | 171 +++++++++++++++++++++++++++++++ docs/source/autoscaling.rst | 4 +- docs/source/getting_started.rst | 82 +++++++++++---- docs/source/index.rst | 14 ++- docs/source/pre_post_install.rst | 6 +- docs/source/toc.rst | 13 +++ 8 files changed, 277 insertions(+), 27 deletions(-) create mode 100644 cli/cfncluster/easyconfig.py create mode 100644 docs/source/toc.rst diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index 3148e55291..1a423a2129 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -119,7 +119,8 @@ def __init__(self, args): except ConfigParser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it - if (__args_func == 'create' or __args_func == 'update') and self.__sanity_check is True: + __args_func = self.args.func.func_name + if (__args_func == 'create' or __args_func == 'update' or __args_func == 'configure') and self.__sanity_check is True: pass else: self.__sanity_check = False diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index 7e3f9d82a1..0799892364 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -1,4 +1,4 @@ -# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +cli/cfncluster/cfnconfig.py# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance with the # License. A copy of the License is located at @@ -16,10 +16,14 @@ import json import cfncluster +import easyconfig def create(args): cfncluster.create(args) +def configure(args): + easyconfig.configure(args) + def status(args): cfncluster.status(args) @@ -121,9 +125,12 @@ def main(): help='show the status of cfncluster with the provided name.') pinstances.set_defaults(func=instances) + pconfigure = subparsers.add_parser('configure', help='creating initial cfncluster configuration') + pconfigure.set_defaults(func=configure) + pversion = subparsers.add_parser('version', help='display version of cfncluster') pversion.set_defaults(func=version) args = parser.parse_args() logging.debug(args) - args.func(args) \ No newline at end of file + args.func(args) diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py new file mode 100644 index 0000000000..bd49158d28 --- /dev/null +++ b/cli/cfncluster/easyconfig.py @@ -0,0 +1,171 @@ +# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Amazon Software License (the 'License'). You may not use this file except in compliance with the +# License. A copy of the License is located at +# +# http://aws.amazon.com/asl/ +# +# or in the 'LICENSE.txt' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +# limitations under the License. + +import ConfigParser +import sys +import boto.ec2 +import boto.vpc +import os +import logging + +import cfnconfig + +logger = logging.getLogger('cfncluster.cfncluster') + +def prompt(prompt, default_value=None, hidden=False, options=None): + if hidden and default_value is not None: + user_prompt = prompt + ' [*******' + default_value[-4:] + ']: ' + else: + user_prompt = prompt + ' [' + if default_value is not None: + user_prompt = user_prompt + default_value + ']: ' + else: + user_prompt = user_prompt + ']: ' + + if isinstance(options, list): + print 'Acceptable Values for %s: ' % prompt + for o in options: + print ' %s' % o + + var = raw_input(user_prompt) + + if var == '': + return default_value + else: + return var + +def get_regions(): + regions = boto.ec2.regions() + names = [] + for region in regions: + names.append(region.name) + return names + +def ec2_conn(aws_access_key_id, aws_secret_access_key, aws_region_name): + if aws_region_name: + region = aws_region_name + elif os.environ.get('AWS_DEFAULT_REGION'): + region = os.environ.get('AWS_DEFAULT_REGION') + else: + region = 'us-east-1' + + conn = boto.ec2.connect_to_region(region,aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key) + return conn + +def vpc_conn(aws_access_key_id, aws_secret_access_key, aws_region_name): + if aws_region_name: + region = aws_region_name + elif os.environ.get('AWS_DEFAULT_REGION'): + region = os.environ.get('AWS_DEFAULT_REGION') + else: + region = 'us-east-1' + + conn = boto.vpc.connect_to_region(region,aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key) + return conn + +def list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name): + conn = ec2_conn(aws_access_key_id, aws_secret_access_key, aws_region_name) + keypairs = conn.get_all_key_pairs() + keynames = [] + for key in keypairs: + keynames.append(key.name) + + if not keynames: + print 'ERROR: No keys found in region ' + aws_region_name + print 'Please create an EC2 keypair before continuing' + sys.exit(1) + + return keynames + +def list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name): + conn = vpc_conn(aws_access_key_id, aws_secret_access_key, aws_region_name) + vpcs = conn.get_all_vpcs() + vpcids = [] + for vpc in vpcs: + vpcids.append(vpc.id) + + if not vpcids: + print 'ERROR: No vpcs found in region ' + aws_region_name + print 'Please create an EC2 vpcpair before continuing' + sys.exit(1) + + return vpcids + +def list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_id): + conn = vpc_conn(aws_access_key_id, aws_secret_access_key, aws_region_name) + subnets = conn.get_all_subnets(filters=[('vpcId', vpc_id)]) + subnetids = [] + for subnet in subnets: + subnetids.append(subnet.id) + + if not subnetids: + print 'ERROR: No subnets found in region ' + aws_region_name + print 'Please create an EC2 subnetpair before continuing' + sys.exit(1) + + return subnetids + +def configure(args): + + # Determine config file name based on args or default + if args.config_file is not None: + config_file = args.config_file + else: + config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config')) + + config = ConfigParser.ConfigParser() + + # Check if configuration file exists + if os.path.isfile(config_file): + config.read(config_file) + config_read = True + + # Prompt for required values, using existing as defaults + cluster_template = prompt('Cluster Name', config.get('global', 'cluster_template') if config.has_option('global', 'cluster_template') else 'mycluster') + aws_access_key_id = prompt('AWS Access Key ID', config.get('aws', 'aws_access_key_id') if config.has_option('aws', 'aws_access_key_id') else None, True) + aws_secret_access_key = prompt('AWS Secret Access Key ID', config.get('aws', 'aws_secret_access_key') if config.has_option('aws', 'aws_secret_access_key') else None, True) + + # Use built in boto regions as an available option + aws_region_name = prompt('AWS Region ID', config.get('aws', 'aws_region_name') if config.has_option('aws', 'aws_region_name') else None, options=get_regions()) + vpcname = prompt('VPC Name', config.get('cluster ' + cluster_template, 'vpc_settings') if config.has_option('cluster ' + cluster_template, 'vpc_settings') else 'myvpc') + + # Query EC2 for available keys as options + key_name = prompt('Key Name', config.get('cluster ' + cluster_template, 'key_name') if config.has_option('cluster ' + cluster_template, 'key_name') else None, options=list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name)) + vpc_id = prompt('VPC ID', config.get('vpc ' + vpcname, 'vpc_id') if config.has_option('vpc ' + vpcname, 'vpc_id') else None, options=list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name)) + master_subnet_id = prompt('Master Subnet ID', config.get('vpc ' + vpcname, 'master_subnet_id') if config.has_option('vpc ' + vpcname, 'master_subnet_id') else None, options=list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_id)) + + # Dictionary of values we want to set + s_global = { '__name__': 'global', 'cluster_template': cluster_template, 'update_check': 'true', 'sanity_check': 'true' } + s_aws = { '__name__': 'aws', 'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key, 'aws_region_name': aws_region_name } + s_cluster = { '__name__': 'cluster ' + cluster_template, 'key_name': key_name, 'vpc_settings': vpcname } + s_vpc = { '__name__': 'vpc ' + vpcname, 'vpc_id': vpc_id, 'master_subnet_id': master_subnet_id } + + sections = [s_global, s_aws, s_cluster, s_vpc] + + # Loop through the configuration sections we care about + for section in sections: + try: + config.add_section(section['__name__']) + except ConfigParser.DuplicateSectionError: + pass + for key, value in section.iteritems(): + # Only update configuration if not set + if value is not None and key is not '__name__': + config.set(section['__name__'], key, value) + + # Write configuration to disk + cfgfile = open(config_file,'w') + config.write(cfgfile) + cfgfile.close() + + # Verify the configuration + cfnconfig.CfnClusterConfig(args) + diff --git a/docs/source/autoscaling.rst b/docs/source/autoscaling.rst index 9e306bed8f..1e5ba1d3db 100644 --- a/docs/source/autoscaling.rst +++ b/docs/source/autoscaling.rst @@ -40,9 +40,9 @@ true. This will cause the ComputeFleet ASG to have the same value for minimum, maximum and desired capacity. References -========== +---------- 1. http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/publishingMetrics.html 2. http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/AutoScalingBehavior.InstanceTermination.html 3. https://github.com/awslabs/cfncluster/tree/master/node/src/nodewatcher -4. http://docs.aws.amazon.com/AutoScaling/latest/APIReference/API_TerminateInstanceInAutoScalingGroup.html \ No newline at end of file +4. http://docs.aws.amazon.com/AutoScaling/latest/APIReference/API_TerminateInstanceInAutoScalingGroup.html diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 0cfc2ce06d..578d959599 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -1,5 +1,8 @@ .. _getting_started: +.. toctree:: + :maxdepth: 2 + ############################### Getting started with cfncluster ############################### @@ -62,33 +65,76 @@ Once installed you will need to setup some initial config. The easiest way to do :: - $ cfncluster create mycluster - Starting: mycluster - Default config /home/ec2-user/.cfncluster/config not found - You can copy a template from here: /usr/lib/python2.6/site-packages/cfncluster/examples/config - $ - $ cp /usr/lib/python2.6/site-packages/cfncluster/examples/config ~/.cfncluster + $ cfncluster configure + +This configure wizard will prompt you for everything you need to create your cluster. You will first be prompted for your cluster name, which is the logical name of your cluster. -You should now edit the config and set some defaults before launching the cluster. First define a keypair that already exists in EC2. If you do not already have a keypair, refer to the EC2 documentation on EC2 Key Pairs - http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html +:: + + Cluster Name [mycluster]: -Then you should associate your choosen keypair with the cluster template. +Next, you will be prompted for your AWS Access & Secret Keys. You can leave these blank to use keys defined in your environment variaables or aws config. Othewise, set them here to be used by cfncluster. :: - [cluster default] - # Name of an existing EC2 KeyPair to enable SSH access to the instances. - key_name = mykey + AWS Access Key ID []: + AWS Secret Access Key ID []: -Next, a simple cluster launches into a VPC and uses an existing subnet which supports public IP's i.e. the route table for the subnet is 0.0.0.0/0 => igw-xxxxxx. The VPC must have "DNS Resolution = yes" and "DNS Hostnames = yes". It should also have DHCP options with the correct "domain-name" for the region, as defined in the docs: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html +Now, you will be presented with a list of valid AWS region identifiers. Choose the region in which you'd like your cluster to run. + +:: + + Acceptable Values for AWS Region ID: + us-east-1 + cn-north-1 + ap-northeast-1 + eu-west-1 + ap-southeast-1 + ap-southeast-2 + us-west-2 + us-gov-west-1 + us-west-1 + eu-central-1 + sa-east-1 + AWS Region ID []: + +Choose a descriptive name for your VPC. Typically, this will something like "production" or "test". + +:: + + VPC Name [myvpc]: + +Next, you will need to choose a keypair that already exists in EC2 in order to log into your master instance. If you do not already have a keypair, refer to the EC2 documentation on EC2 Key Pairs - http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html + +:: + + Acceptable Values for Key Name: + keypair1 + keypair-test + production-key + Key Name []: + +Choose the VPC ID in which you'd like your cluster launched into. + +:: + + Acceptable Values for VPC ID: + vpc-1kd24879 + vpc-blk4982d + VPC ID []: + +Finally, choose the subnet in which you'd like your master server to run in. :: - ## VPC Settings - [vpc public] - # ID of the VPC you want to provision cluster into. - vpc_id = CHANGE ME, for example vpc-a1b2c3d4 - # ID of the Subnet you want to provision the Master server into - master_subnet_id = CHANGE ME, for exaple subnet-1ab2c3d4 + Acceptable Values for Master Subnet ID: + subnet-9k284a6f + subnet-1k01g357 + subnet-b921nv04 + Master Subnet ID []: + + +Next, a simple cluster launches into a VPC and uses an existing subnet which supports public IP's i.e. the route table for the subnet is 0.0.0.0/0 => igw-xxxxxx. The VPC must have "DNS Resolution = yes" and "DNS Hostnames = yes". It should also have DHCP options with the correct "domain-name" for the region, as defined in the docs: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html Once all of those settings contain valid values, you can launch the cluster by repeating the command that was used at the start. diff --git a/docs/source/index.rst b/docs/source/index.rst index dddc165c04..729cbade81 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,9 +3,21 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. + Welcome to cfncluster's documentation! ====================================== +.. toctree:: + :hidden: + + getting_started + networking + aws_services + autoscaling + pre_post_install + s3_resources + toc + cfncluster is a framework that deploys and maintains HPC clusters on AWS. It is reasonably agnostic to what the cluster is for and can easily be extended to support different frameworks. The CLI is stateless, everything is done using CloudFormation or resources within AWS. Getting Started @@ -38,4 +50,4 @@ Indices and tables * :ref:`genindex` * :ref:`search` - +* :ref:`toc` diff --git a/docs/source/pre_post_install.rst b/docs/source/pre_post_install.rst index d3c04bda63..36cdbc0aaf 100644 --- a/docs/source/pre_post_install.rst +++ b/docs/source/pre_post_install.rst @@ -14,7 +14,7 @@ Arguments can be passed to scripts by specifying them in the config. These will If a pre/post-install actions fails, then the instance bootstrap will be considered failed and it will not continue. Success is signalled with an exit code of 0, any other exit code will be considered a fail. Configuration -============= +------------- The following config settings are used to define pre/post-install actions and arguments. All options are optional and are not required for basic cluster install. @@ -34,7 +34,7 @@ The following config settings are used to define pre/post-install actions and ar post_install_args = NONE Example -======= +------- The following are some steps to create a simple post install script that installs the R packages in a cluster. @@ -60,4 +60,4 @@ The following are some steps to create a simple post install script that install 4. Lauch a cluster -``cfncluster create mycluster`` \ No newline at end of file +``cfncluster create mycluster`` diff --git a/docs/source/toc.rst b/docs/source/toc.rst new file mode 100644 index 0000000000..d7bf83a261 --- /dev/null +++ b/docs/source/toc.rst @@ -0,0 +1,13 @@ +.. _toc: + +Table of Contents +================= + +.. toctree:: + + getting_started + networking + aws_services + autoscaling + pre_post_install + s3_resources From a5ffb786c85b8347985a6676e2447186f8467d41 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Thu, 5 Feb 2015 15:02:26 -0800 Subject: [PATCH 10/13] Fixed typo --- cli/cfncluster/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index 0799892364..1c5d2391e1 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -1,4 +1,4 @@ -cli/cfncluster/cfnconfig.py# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance with the # License. A copy of the License is located at From 8f674dd0b6ba59e0528441cc14e5c065db254bcb Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Wed, 18 Feb 2015 12:39:17 -0800 Subject: [PATCH 11/13] syntax cleanup --- cli/cfncluster/easyconfig.py | 6 +++--- node/nodewatcher/nodewatcher.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index bd49158d28..d4d6f81f59 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -129,13 +129,13 @@ def configure(args): config_read = True # Prompt for required values, using existing as defaults - cluster_template = prompt('Cluster Name', config.get('global', 'cluster_template') if config.has_option('global', 'cluster_template') else 'mycluster') + cluster_template = prompt('Cluster Name', config.get('global', 'cluster_template') if config.has_option('global', 'cluster_template') else 'default') aws_access_key_id = prompt('AWS Access Key ID', config.get('aws', 'aws_access_key_id') if config.has_option('aws', 'aws_access_key_id') else None, True) aws_secret_access_key = prompt('AWS Secret Access Key ID', config.get('aws', 'aws_secret_access_key') if config.has_option('aws', 'aws_secret_access_key') else None, True) # Use built in boto regions as an available option aws_region_name = prompt('AWS Region ID', config.get('aws', 'aws_region_name') if config.has_option('aws', 'aws_region_name') else None, options=get_regions()) - vpcname = prompt('VPC Name', config.get('cluster ' + cluster_template, 'vpc_settings') if config.has_option('cluster ' + cluster_template, 'vpc_settings') else 'myvpc') + vpcname = prompt('VPC Name', config.get('cluster ' + cluster_template, 'vpc_settings') if config.has_option('cluster ' + cluster_template, 'vpc_settings') else 'public') # Query EC2 for available keys as options key_name = prompt('Key Name', config.get('cluster ' + cluster_template, 'key_name') if config.has_option('cluster ' + cluster_template, 'key_name') else None, options=list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name)) @@ -148,7 +148,7 @@ def configure(args): s_cluster = { '__name__': 'cluster ' + cluster_template, 'key_name': key_name, 'vpc_settings': vpcname } s_vpc = { '__name__': 'vpc ' + vpcname, 'vpc_id': vpc_id, 'master_subnet_id': master_subnet_id } - sections = [s_global, s_aws, s_cluster, s_vpc] + sections = [s_aws, s_cluster, s_vpc, s_global] # Loop through the configuration sections we care about for section in sections: diff --git a/node/nodewatcher/nodewatcher.py b/node/nodewatcher/nodewatcher.py index fae4c64876..433e6ef097 100755 --- a/node/nodewatcher/nodewatcher.py +++ b/node/nodewatcher/nodewatcher.py @@ -97,7 +97,7 @@ def getJobs(s,hostname): return _jobs -def selfTerminate(region,asg,instance_id): +def selfTerminate(region, asg, instance_id): _as_conn = boto.ec2.autoscale.connect_to_region(region,proxy=boto.config.get('Boto', 'proxy'), proxy_port=boto.config.get('Boto', 'proxy_port')) _asg = _as_conn.get_all_groups(names=[asg])[0] @@ -126,7 +126,7 @@ def main(): print('Percent of hour used: %d' % hour_percentile) if hour_percentile > 95: - selfTerminate(region,asg,instance_id) + selfTerminate(region, asg, instance_id) if __name__ == "__main__": main() From 9692e33c38aa25589721b5dc7aa3965cf77bf0d6 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Wed, 18 Feb 2015 11:36:05 -0800 Subject: [PATCH 12/13] Updated Documentation. --- docs/source/configuration.rst | 479 +++++++++++++++++++++++++++++++ docs/source/functional.rst | 11 + docs/source/index.rst | 22 +- docs/source/networking.rst | 4 +- docs/source/pre_post_install.rst | 4 +- docs/source/s3_resources.rst | 2 +- docs/source/toc.rst | 13 - docs/source/welcome.rst | 10 + docs/staging/first_job.rst | 11 + 9 files changed, 525 insertions(+), 31 deletions(-) create mode 100644 docs/source/configuration.rst create mode 100644 docs/source/functional.rst delete mode 100644 docs/source/toc.rst create mode 100644 docs/source/welcome.rst create mode 100644 docs/staging/first_job.rst diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst new file mode 100644 index 0000000000..0cee6b3599 --- /dev/null +++ b/docs/source/configuration.rst @@ -0,0 +1,479 @@ +Configuration +============= +.. toctree:: + +cfncluster uses the file ``~/.cfncluster/config`` by default for all configuration parameters. + +You can see an example configuration file ``site-packages/cfncluster/examples/config`` + +Layout +------ + +Configuration is defined in multiple sections. Required sections are "global", "aws", one "cluster", and one "subnet". + +A section starts with the section name in brackets, followed by parameters and configuration. :: + + [global] + cluster_template = default + update_check = true + sanity_check = true + + +Configuration Options +--------------------- + +global +^^^^^^ +Global configuration options related to cfncluster. :: + + [global] + +cluster_template +"""""""""""""""" +The name of the cluster section used for the cluster. + +See the :ref:`Cluster Definition `. :: + + cluster_template = default + +update_check +"""""""""""" +Whether or not to check for updates to cfncluster. :: + + update_check = true + +sanity_check +"""""""""""" +Attempts to validate that resources defined in parameters actually exist. :: + + sanity_check = true + +aws +^^^ +This is the AWS credentials section (required). These settings apply to all clusters. + +If not defined, boto will attempt to use a) enviornment or b) EC2 IAM role. :: + + [aws] + aws_access_key_id = #your_aws_access_key_id + aws_secret_access_key = #your_secret_access_key + + # Defaults to us-east-1 if not defined in enviornment or below + aws_region_name = #region + +.. _cluster_definition: + +cluster +^^^^^^^ +You can define one or more clusters for different types of jobs or workloads. + +Each cluster has it's own configuration based on your needs. + +The format is [cluster ]. :: + + [cluster default] + +key_name +"""""""" +Name of an existing EC2 KeyPair to enable SSH access to the instances. :: + + key_name = mykey + +template_url +"""""""""""" +Overrides the path to the cloudformation template used to create the cluster + +Defaults to ``https://s3.amazonaws.com/cfncluster-/templates/cfncluster-.cfn.json``. :: + + template_url = https://s3.amazonaws.com/cfncluster-us-east-1/templates/cfncluster.cfn.json + +compute_instance_type +""""""""""""""""""""" +The EC2 instance type used for the cluster compute nodes. + +Defaults to t2.micro for default template. :: + + compute_instance_type = t2.micro + +master_instance_type +"""""""""""""""""""" +The EC2 instance type use for the master node. + +This defaults to t2.micro for default template. :: + + master_instance_type = t2.micro + +initial_queue_size +"""""""""""""""""" +The inital number of EC2 instances to launch as compute nodes in the cluster. + +The default is 2 for default template. :: + + initial_queue_size = 2 + +max_queue_size +"""""""""""""" +The maximum number of EC2 instances that can be launched in the cluster. + +This defaults to 10 for the default template. :: + + max_queue_size = 10 + +maintain_initial_size +""""""""""""""""""""" +Boolean flag to set autoscaling group to maintain initial size. + +If set to true, the autoscaling group can scale down to 0 compute instances if there are no jobs for a period of time. + +Defaults to false for the default template. :: + + maintain_initial_size = false + +scheduler +""""""""" +Cluster scheduler + +Defaults to sge for the default template. :: + + scheduler = sge + +cluster_type +"""""""""""" +Type of cluster to launch i.e. ondemand or spot + +Defaults to ondemand for the default template. :: + + cluster_type = ondemand + +spot_price +""""""""""" +If cluster_type is set to spot, the maximum spot price for the ComputeFleet. :: + + spot_price = 0.00 + +custom_ami +"""""""""" +ID of a Custom AMI, to use instead of default published AMI's. :: + + custom_ami = NONE + +s3_read_resource +"""""""""""""""" +Specify S3 resource which cfncluster nodes will be granted read-only access + +See :doc:`working with S3 ` for details on format. + +Defaults to NONE for the default template. :: + + s3_read_resource = NONE + +s3_read_write_resource +"""""""""""""""""""""" +Specify S3 resource which cfncluster nodes will be granted read-write access + +See :doc:`working with S3 ` for details on format. + +Defaults to NONE for the default template. :: + + s3_read_write_resource = NONE + +pre_install +""""""""""" +URL to a preinstall script. This is executed before any of the boot_as_* scripts are run + +Can be specified in "http://hostname/path/to/script.sh" or "s3://bucketname/path/to/script.sh" format. + +Defaults to NONE for the default template. :: + + pre_install = NONE + +pre_install_args +"""""""""""""""" +Quoted list of arguments to be passed to preinstall script + +Defaults to NONE for the default template. :: + + pre_install_args = NONE + +post_install +"""""""""""" +URL to a postinstall script. This is executed after any of the boot_as_* scripts are run + +Can be specified in "http://hostname/path/to/script.sh" or "s3://bucketname/path/to/script.sh" format. + +Defaults to NONE for the default template. :: + + post_install = NONE + +post_install_args +""""""""""""""""" +Arguments to be passed to postinstall script + +Defaults to NONE for the default template. :: + + post_install_args = NONE + +proxy_server +"""""""""""" +HTTP(S) proxy server, typically http://x.x.x.x:8080 + +Defaults to NONE for the default template. :: + + proxy_server = NONE + +placement_group +""""""""""""""" +Cluster placement group. This placement group must already exist. + +Defaults to NONE for the default template. :: + + placement_group = NONE + +placement +""""""""" +Cluster placment logic. This enables the whole cluster or only compute to use the placement group. + +Defaults to cluster in the default template. :: + + placement = cluster + +ephemeral_dir +""""""""""""" +If instance store volumes exist, this is the path/mountpoint they will be mounted on. + +Defaults to /scratch in the default template. :: + + ephemeral_dir = /scratch + +shared_dir +"""""""""" +Path/mountpoint for shared EBS volume + +Defaults to /shared in the default template. :: + + shared_dir = /shared + +encrypted_ephemeral +""""""""""""""""""" +Encrypted ephemeral drives. In-memory keys, non-recoverable. + +Defaults to false in default template. :: + + encrypted_ephemeral = false + +base_os +""""""" +OS type used in the cluster + +Defaults to centos6 in the default template. :: + + base_os = centos6 + +cwl_region +"""""""""" +CloudWatch Logs region + +Defaults to NONE in the default template. :: + + cwl_region = NONE + +cwl_log_group +""""""""""""" +CloudWatch Logs Log Group name + +Defaults to NONE in the default template. :: + + cwl_log_group = NONE + +vpc_settings +"""""""""""" +Settings section relating to VPC to be used + +See :ref:`VPC Section `. :: + + vpc_settings = public + +ebs_settings +"""""""""""" +Settings section relating to EBS volume mounted on the master. + +See :ref:`EBS Section `. :: + + ebs_settings = custom + +scaling +""""""" +Settings section relation to scaling + +See :ref:`Scaling Section `. :: + + scaling = custom + +.. _vpc_section: + +vpc +^^^ +VPC Configuration Settings:: + + [vpc public] + vpc_id = vpc-xxxxxx + master_subnet_id = subnet-xxxxxx + +vpc_id +"""""" +ID of the VPC you want to provision cluster into.:: + + vpc_id = vpc-xxxxxx + +master_subnet_id +"""""""""""""""" +ID of an existing subnet you want to provision the Master server into. :: + + master_subnet_id = subnet-xxxxxx + +ssh_from +"""""""" +CIDR formatted IP range in which to allow SSH access from. + +This is only used when cfncluster creates the security group. + +Defaults to 0.0.0.0/0 in the default template. :: + + ssh_from = 0.0.0.0/0 + +additional_sg +""""""""""""" +Additional VPC security group Id for all instances. + +Defaults to NONE in the default template. :: + + additional_sg = sg-xxxxxx + +master_subnet_id +"""""""""""""""" +ID of an existing subnet you want to provision the compute nodes into. :: + + master_subnet_id = subnet-xxxxxx + +compute_subnet_cidr +""""""""""""""""""" +If you wish for cfncluster to create a compute subnet, this is the CIDR that. :: + + compute_subnet_cidr = 10.0.100.0/24 + +.. _ebs_section: + +ebs +^^^ +EBS Volume configuration settings for the volume mounted on the master node and shared via NFS to compute nodes. :: + + [ebs custom] + ebs_snapshot_id = snap-xxxxx + volume_type = io1 + volume_iops = 200 + +ebs_snapshot_id +""""""""""""""" +Id of EBS snapshot if using snapshot as source for volume. + +Defaults to NONE for default template. :: + + ebs_snapshot_id = snap-xxxxx + +volume_type +""""""""""" +The `API name `_ for the type of volume you wish to launch. + +Defaults to gp2 for default template. :: + + volume_type = io1 + +volume_size +""""""""""" +Size of volume to be created (if not using a snapshot). + +Defaults to 20GB for default template. :: + + volume_size = 20 + +volume_iops +""""""""""" +Number of IOPS for io1 type volumes. :: + + volume_iops = 200 + +encrypted +""""""""" +Whether or not the volume should be encrypted (should not be used with snapshots). + +Defaults to false for default template. :: + + encrypted = false + +.. _scaling_section: + +scaling +^^^^^^^ +Settings which define how the compute nodes scale. :: + + + [scaling custom] + scaling_period = 60 + scaling_cooldown = 120 + +scaling_threshold +""""""""""""""""" +Threshold for triggering CloudWatch ScaleUp action. + +Defaults to 4 for default template. :: + + scaling_threshold = 4 + +scaling_adjustment +"""""""""""""""""" +Number of instances to add when called CloudWatch ScaleUp action. + +Defaults to 2 for default template. :: + + scaling_adjustment = 2 + + +scaling_threshold2 +"""""""""""""""""" +Threshold for triggering CloudWatch ScaleUp2 action. + +Defaults to 200 for default template. :: + + scaling_threshold2 = 200 + +scaling_adjustment2 +""""""""""""""""""" +Number of instances to add when called CloudWatch ScaleUp2 action + +Defaults to 20 for default template. :: + + scaling_adjustment2 = 20 + +scaling_period +"""""""""""""" +Period to measure ScalingThreshold. + +Defaults to 60 for default template. :: + + scaling_period = 60 + +scaling_evaluation_periods +"""""""""""""""""""""""""" +Number of periods to measure ScalingThreshold. + +Defaults to 2 for default template. :: + + scaling_evaluation_periods = 2 + +scaling_cooldown +"""""""""""""""" +Amount of time in seconds to wait before attempting further scaling actions. + +Defaults to 120 for the default template. :: + + scaling_cooldown = 120 diff --git a/docs/source/functional.rst b/docs/source/functional.rst new file mode 100644 index 0000000000..d41aa23926 --- /dev/null +++ b/docs/source/functional.rst @@ -0,0 +1,11 @@ +.. _functional + +How cfncluster Works +#################### + +cfncluster was built not only as a way to manage clusters, but as a reference on how to use AWS services to build your HPC environment + +.. toctree:: + + aws_services + autoscaling diff --git a/docs/source/index.rst b/docs/source/index.rst index 729cbade81..df03ac6b83 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -4,21 +4,18 @@ contain the root `toctree` directive. -Welcome to cfncluster's documentation! -====================================== +cfncluster +########## -.. toctree:: - :hidden: +cfncluster is a framework that deploys and maintains High Performance Clusters (HPC) on AWS. It is reasonably agnostic to what the cluster is for and can easily be extended to support different frameworks. The CLI is stateless, everything is done using CloudFormation or resources within AWS. - getting_started - networking - aws_services - autoscaling - pre_post_install - s3_resources - toc +.. toctree:: + :maxdepth: 2 -cfncluster is a framework that deploys and maintains HPC clusters on AWS. It is reasonably agnostic to what the cluster is for and can easily be extended to support different frameworks. The CLI is stateless, everything is done using CloudFormation or resources within AWS. + getting_started + welcome + configuration + functional Getting Started --------------- @@ -50,4 +47,3 @@ Indices and tables * :ref:`genindex` * :ref:`search` -* :ref:`toc` diff --git a/docs/source/networking.rst b/docs/source/networking.rst index e653861a7e..4b72dcff0f 100644 --- a/docs/source/networking.rst +++ b/docs/source/networking.rst @@ -1,7 +1,7 @@ .. _networking: -cfncluster Networking Configurations -==================================== +Network Configurations +====================== cfncluster leverages Amazon Virtual Private Cloud(VPC) for networking. This provides a very flexiable and configurable networking platform to deploy clusters within. cfncluster support the following high-level configurations: diff --git a/docs/source/pre_post_install.rst b/docs/source/pre_post_install.rst index 36cdbc0aaf..b9d28bede0 100644 --- a/docs/source/pre_post_install.rst +++ b/docs/source/pre_post_install.rst @@ -1,7 +1,7 @@ .. _pre_post_install: -Pre & Post Install Actions -========================== +Custom Bootstrap Actions +======================== cfncluster can execute arbritary code either before(pre) or after(post) the main bootstrap action during cluster creation. This code is typically stored in S3 and accessed via HTTP(S) during cluster creation. The code will be executed as root and can be in any script language supppoted by the cluster OS, typically `bash` or `python`. diff --git a/docs/source/s3_resources.rst b/docs/source/s3_resources.rst index 4e22823f0a..867a3b3e6d 100644 --- a/docs/source/s3_resources.rst +++ b/docs/source/s3_resources.rst @@ -1,6 +1,6 @@ .. _s3_resources: -working with S3 +Working with S3 =============== Accessing S3 within cfncluster can be controlled through two parameters in the cfncluster config. diff --git a/docs/source/toc.rst b/docs/source/toc.rst deleted file mode 100644 index d7bf83a261..0000000000 --- a/docs/source/toc.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _toc: - -Table of Contents -================= - -.. toctree:: - - getting_started - networking - aws_services - autoscaling - pre_post_install - s3_resources diff --git a/docs/source/welcome.rst b/docs/source/welcome.rst new file mode 100644 index 0000000000..a2d632b67f --- /dev/null +++ b/docs/source/welcome.rst @@ -0,0 +1,10 @@ +.. _welcome + +Working with cfncluster +####################### + +.. toctree:: + + networking + pre_post_install + s3_resources diff --git a/docs/staging/first_job.rst b/docs/staging/first_job.rst new file mode 100644 index 0000000000..011e4354c1 --- /dev/null +++ b/docs/staging/first_job.rst @@ -0,0 +1,11 @@ +Running your first job +###################### + + +- Run through getting started to create your cluster +- SSH to master node +- Download source +- Compile +- Create job file +- Submit job +- View output From 9e9f0d8ecb118ec9335a849976797f427b15a6f8 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Wed, 18 Feb 2015 18:40:46 -0800 Subject: [PATCH 13/13] Push for v0.0.19 release --- CHANGELOG.rst | 7 ++++++ amis.txt | 20 ++++++++--------- cli/cfncluster/easyconfig.py | 2 +- cli/setup.py | 4 ++-- cloudformation/cfncluster.cfn.json | 36 +++++++++++++++--------------- 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d6150aed34..085c197fa4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ CHANGELOG ========= +0.0.19 +====== +* feature:``cli``: Added configure command; easy config setup +* updates:``docs``: Addtional documentation for configuration options +* updates:``ami``: Pulled latest CentOS6 errata +* bugfix:``cfncluster``: Fixed issue with nodewatcher not scaling down + 0.0.18 ====== * updates:``ami``: Custom CentOS 6 kernel repo added, support for >32 vCPUs diff --git a/amis.txt b/amis.txt index 83030df39f..87d31ad580 100644 --- a/amis.txt +++ b/amis.txt @@ -1,10 +1,10 @@ -us-west-2 ami-87de82b7 -eu-central-1 ami-8c97a791 -sa-east-1 ami-2bdd6036 -ap-northeast-1 ami-a8d8c8a9 -eu-west-1 ami-f738be80 -us-east-1 ami-4ad6a522 -us-west-1 ami-298e916c -ap-southeast-2 ami-efc4aed5 -ap-southeast-1 ami-705a7222 -us-gov-west-1 ami-5fcfa97c +us-west-2 ami-ab7f599b +eu-central-1 ami-b43705a9 +sa-east-1 ami-818b359c +ap-northeast-1 ami-891bfb89 +eu-west-1 ami-87fd69f0 +us-east-1 ami-7660361e +us-west-1 ami-86b0aac3 +ap-southeast-2 ami-7d730547 +ap-southeast-1 ami-fafcc8a8 +us-gov-west-1 ami-5d68097e diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index d4d6f81f59..fe0798f157 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -129,7 +129,7 @@ def configure(args): config_read = True # Prompt for required values, using existing as defaults - cluster_template = prompt('Cluster Name', config.get('global', 'cluster_template') if config.has_option('global', 'cluster_template') else 'default') + cluster_template = prompt('Cluster Template', config.get('global', 'cluster_template') if config.has_option('global', 'cluster_template') else 'default') aws_access_key_id = prompt('AWS Access Key ID', config.get('aws', 'aws_access_key_id') if config.has_option('aws', 'aws_access_key_id') else None, True) aws_secret_access_key = prompt('AWS Secret Access Key ID', config.get('aws', 'aws_secret_access_key') if config.has_option('aws', 'aws_secret_access_key') else None, True) diff --git a/cli/setup.py b/cli/setup.py index edce590878..1a1bebd152 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -20,8 +20,8 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() console_scripts = ['cfncluster = cfncluster.cli:main'] -version = "0.0.99" -requires = ['boto>=2.35.1'] +version = "0.0.19" +requires = ['boto>=2.36'] if sys.version_info[:2] == (2, 6): # For python2.6 we have to require argparse since it diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 17a3519945..0c9cee98d3 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -720,35 +720,35 @@ } }, "AWSRegionOS2AMI" : { - "eu-west-1" : { - "centos6" : "ami-f738be80" + "us-west-2" : { + "centos6" : "ami-ab7f599b" }, - "us-east-1" : { - "centos6" : "ami-4ad6a522" + "eu-central-1" : { + "centos6" : "ami-b43705a9" + }, + "sa-east-1" : { + "centos6" : "ami-818b359c" }, "ap-northeast-1" : { - "centos6" : "ami-a8d8c8a9" + "centos6" : "ami-891bfb89" }, - "us-west-2" : { - "centos6" : "ami-87de82b7" + "eu-west-1" : { + "centos6" : "ami-87fd69f0" }, - "sa-east-1" : { - "centos6" : "ami-2bdd6036" + "us-east-1" : { + "centos6" : "ami-7660361e" }, "us-west-1" : { - "centos6" : "ami-298e916c" - }, - "ap-southeast-1" : { - "centos6" : "ami-705a7222" + "centos6" : "ami-86b0aac3" }, "ap-southeast-2" : { - "centos6" : "ami-efc4aed5" + "centos6" : "ami-7d730547" }, - "eu-central-1" : { - "centos6" : "ami-8c97a791" + "ap-southeast-1" : { + "centos6" : "ami-fafcc8a8" }, "us-gov-west-1" : { - "centos6" : "ami-5fcfa97c" + "centos6" : "ami-5d68097e" } }, "ClusterUser" : { @@ -1819,7 +1819,7 @@ { "Fn::GetAtt" : [ "MasterServer", - "PrivateIp" + "PrivateDnsName" ] }, "\n",