diff --git a/chalice/awsclient.py b/chalice/awsclient.py index c63ea42f4..67e2c7052 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -397,6 +397,7 @@ def create_function( xray: Optional[bool] = None, timeout: OptInt = None, memory_size: OptInt = None, + ephemeral_storage: OptInt = None, security_group_ids: OptStrList = None, subnet_ids: OptStrList = None, layers: OptStrList = None, @@ -419,6 +420,8 @@ def create_function( kwargs['Timeout'] = timeout if memory_size is not None: kwargs['MemorySize'] = memory_size + if ephemeral_storage is not None: + kwargs['EphemeralStorage'] = {'Size': ephemeral_storage} if security_group_ids is not None and subnet_ids is not None: kwargs['VpcConfig'] = self._create_vpc_config( security_group_ids=security_group_ids, @@ -902,6 +905,7 @@ def update_function( xray: Optional[bool] = None, timeout: OptInt = None, memory_size: OptInt = None, + ephemeral_storage: OptInt = None, role_arn: OptStr = None, subnet_ids: OptStrList = None, security_group_ids: OptStrList = None, @@ -921,6 +925,7 @@ def update_function( runtime=runtime, timeout=timeout, memory_size=memory_size, + ephemeral_storage=ephemeral_storage, role_arn=role_arn, xray=xray, subnet_ids=subnet_ids, @@ -967,19 +972,19 @@ def delete_function_concurrency(self, function_name: str) -> None: lambda_client = self._client('lambda') lambda_client.delete_function_concurrency(FunctionName=function_name) - def _update_function_config( + def _assemble_update_function_config_kwargs( self, environment_variables: StrMap, runtime: OptStr, timeout: OptInt, memory_size: OptInt, + ephemeral_storage: OptInt, role_arn: OptStr, subnet_ids: OptStrList, security_group_ids: OptStrList, - function_name: str, layers: OptStrList, xray: Optional[bool], - ) -> None: + ) -> Dict[str, Any]: kwargs: Dict[str, Any] = {} if environment_variables is not None: kwargs['Environment'] = {'Variables': environment_variables} @@ -989,6 +994,8 @@ def _update_function_config( kwargs['Timeout'] = timeout if memory_size is not None: kwargs['MemorySize'] = memory_size + if ephemeral_storage is not None: + kwargs['EphemeralStorage'] = {'Size': ephemeral_storage} if role_arn is not None: kwargs['Role'] = role_arn if xray: @@ -999,6 +1006,34 @@ def _update_function_config( ) if layers is not None: kwargs['Layers'] = layers + return kwargs + + def _update_function_config( + self, + environment_variables: StrMap, + runtime: OptStr, + timeout: OptInt, + memory_size: OptInt, + ephemeral_storage: OptInt, + role_arn: OptStr, + subnet_ids: OptStrList, + security_group_ids: OptStrList, + function_name: str, + layers: OptStrList, + xray: Optional[bool], + ) -> None: + kwargs = self._assemble_update_function_config_kwargs( + environment_variables, + runtime, + timeout, + memory_size, + ephemeral_storage, + role_arn, + subnet_ids, + security_group_ids, + layers, + xray, + ) if kwargs: self._do_update_function_config(function_name, kwargs) diff --git a/chalice/config.py b/chalice/config.py index e1e84ba9e..31abdf718 100644 --- a/chalice/config.py +++ b/chalice/config.py @@ -273,6 +273,12 @@ def lambda_memory_size(self) -> int: varies_per_chalice_stage=True, varies_per_function=True) + @property + def lambda_ephemeral_storage(self) -> int: + return self._chain_lookup('lambda_ephemeral_storage', + varies_per_chalice_stage=True, + varies_per_function=True) + @property def lambda_timeout(self) -> int: return self._chain_lookup('lambda_timeout', diff --git a/chalice/constants.py b/chalice/constants.py index 6dc2021a2..648707193 100644 --- a/chalice/constants.py +++ b/chalice/constants.py @@ -50,6 +50,7 @@ def index(): DEFAULT_LAMBDA_TIMEOUT = 60 DEFAULT_LAMBDA_MEMORY_SIZE = 128 +DEFAULT_LAMBDA_EPHEMERAL_STORAGE = 512 MAX_LAMBDA_DEPLOYMENT_SIZE = 50 * (1024 ** 2) # This is the name of the main handler used to # handle API gateway requests. This is used as a key diff --git a/chalice/deploy/appgraph.py b/chalice/deploy/appgraph.py index f7e9bcb73..d7a0bb38f 100644 --- a/chalice/deploy/appgraph.py +++ b/chalice/deploy/appgraph.py @@ -567,6 +567,7 @@ def _build_lambda_function( tags=config.tags, timeout=config.lambda_timeout, memory_size=config.lambda_memory_size, + ephemeral_storage=config.lambda_ephemeral_storage, deployment_package=deployment, role=role, security_group_ids=security_group_ids, diff --git a/chalice/deploy/deployer.py b/chalice/deploy/deployer.py index de1c17cf7..d6402488c 100644 --- a/chalice/deploy/deployer.py +++ b/chalice/deploy/deployer.py @@ -103,6 +103,7 @@ from chalice.constants import VPC_ATTACH_POLICY from chalice.constants import DEFAULT_LAMBDA_TIMEOUT from chalice.constants import DEFAULT_LAMBDA_MEMORY_SIZE +from chalice.constants import DEFAULT_LAMBDA_EPHEMERAL_STORAGE from chalice.constants import DEFAULT_TLS_VERSION from chalice.constants import SQS_EVENT_SOURCE_POLICY from chalice.constants import KINESIS_EVENT_SOURCE_POLICY @@ -422,10 +423,12 @@ def handle(self, config, resource): class InjectDefaults(BaseDeployStep): def __init__(self, lambda_timeout=DEFAULT_LAMBDA_TIMEOUT, lambda_memory_size=DEFAULT_LAMBDA_MEMORY_SIZE, + lambda_ephemeral_storage=DEFAULT_LAMBDA_EPHEMERAL_STORAGE, tls_version=DEFAULT_TLS_VERSION): - # type: (int, int, str) -> None + # type: (int, int, int, str) -> None self._lambda_timeout = lambda_timeout self._lambda_memory_size = lambda_memory_size + self._lambda_ephemeral_storage = lambda_ephemeral_storage self._tls_version = DEFAULT_TLS_VERSION def handle_lambdafunction(self, config, resource): @@ -434,6 +437,8 @@ def handle_lambdafunction(self, config, resource): resource.timeout = self._lambda_timeout if resource.memory_size is None: resource.memory_size = self._lambda_memory_size + if resource.ephemeral_storage is None: + resource.ephemeral_storage = self._lambda_ephemeral_storage def handle_domainname(self, config, resource): # type: (Config, models.DomainName) -> None diff --git a/chalice/deploy/models.py b/chalice/deploy/models.py index 89a81cae3..8df3402d5 100644 --- a/chalice/deploy/models.py +++ b/chalice/deploy/models.py @@ -196,6 +196,7 @@ class LambdaFunction(ManagedModel): tags: StrMap timeout: int memory_size: int + ephemeral_storage: int role: IAMRole security_group_ids: List[str] subnet_ids: List[str] diff --git a/chalice/deploy/planner.py b/chalice/deploy/planner.py index b7a96b769..fc8cb7ecf 100644 --- a/chalice/deploy/planner.py +++ b/chalice/deploy/planner.py @@ -489,6 +489,7 @@ def _plan_lambdafunction(self, resource): 'tags': resource.tags, 'timeout': resource.timeout, 'memory_size': resource.memory_size, + 'ephemeral_storage': resource.ephemeral_storage, 'security_group_ids': resource.security_group_ids, 'subnet_ids': resource.subnet_ids, 'layers': layers @@ -521,6 +522,7 @@ def _plan_lambdafunction(self, resource): 'tags': resource.tags, 'timeout': resource.timeout, 'memory_size': resource.memory_size, + 'ephemeral_storage': resource.ephemeral_storage, 'security_group_ids': resource.security_group_ids, 'subnet_ids': resource.subnet_ids, 'layers': layers diff --git a/chalice/package.py b/chalice/package.py index ecdd0268e..fd0071e02 100644 --- a/chalice/package.py +++ b/chalice/package.py @@ -246,6 +246,7 @@ def _generate_lambdafunction(self, resource, template): 'Tracing': resource.xray and 'Active' or 'PassThrough', 'Timeout': resource.timeout, 'MemorySize': resource.memory_size, + 'EphemeralStorage': {'Size': resource.ephemeral_storage}, }, } # type: Dict[str, Any] @@ -1243,6 +1244,7 @@ def _generate_lambdafunction(self, resource, template): 'runtime': resource.runtime, 'handler': resource.handler, 'memory_size': resource.memory_size, + 'ephemeral_storage': resource.ephemeral_storage, 'tags': resource.tags, 'timeout': resource.timeout, 'source_code_hash': '${filebase64sha256("%s")}' % ( diff --git a/docs/source/topics/configfile.rst b/docs/source/topics/configfile.rst index 35a498dd2..1f0a68642 100644 --- a/docs/source/topics/configfile.rst +++ b/docs/source/topics/configfile.rst @@ -143,6 +143,14 @@ your function. The default ``lambda_memory_size`` value is ``128``. The value must be a multiple of 64 MB. +``lambda_ephemeral_storage`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An integer representing the amount of ephemeral storage, in MB, your Lambda +function is given. The default ``lambda_ephemeral_storage`` value is ``512``. +The maximum value is ``10240``. + + ``lambda_timeout`` ~~~~~~~~~~~~~~~~~~ @@ -387,6 +395,7 @@ that can be applied per function: * ``iam_role_arn`` * ``lambda_memory_size`` * ``lambda_timeout`` +* ``lambda_ephemeral_storage`` * ``layers`` * ``manage_iam_role`` * ``reserved_concurrency`` @@ -600,6 +609,30 @@ The ``prod`` stage will have these environment variables set:: } +.. _ephemeral-storage-examples: + +Ephemeral Storage +~~~~~~~~~~~~~~~~~ + +In the following example, the ephemeral storage for the +"foo" function will be set to 1024 MB. In the dev stage +that value will be overridden and set to 2048 MB:: + + { + "version": "2.0", + "app_name": "app", + "lambda_functions": { + "foo": { + "lambda_ephemeral_storage": 1024 + } + "dev": { + "lambda_functions": { + "foo": { + "lambda_ephemeral_storage": 2048 + } + } + + Per Lambda Examples ~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/topics/middleware.rst b/docs/source/topics/middleware.rst index 90a8ea0de..465315bc1 100644 --- a/docs/source/topics/middleware.rst +++ b/docs/source/topics/middleware.rst @@ -271,15 +271,15 @@ Integrating with AWS Lambda Powertools -------------------------------------- `AWS Lambda Powertools -`__ is a suite of +`__ is a suite of utilities for AWS Lambda functions that makes tracing with AWS X-Ray, structured logging and creating custom metrics asynchronously easier. You can use Chalice middleware to easily integrate Lambda Powertools with your Chalice apps. In this example, we'll use the `Logger -`__ -and `Tracer `__ +`__ +and `Tracer `__ and convert them to Chalice middleware so they will be automatically applied to all Lambda functions in our application. diff --git a/tests/functional/test_awsclient.py b/tests/functional/test_awsclient.py index 6c15b463f..2468611d1 100644 --- a/tests/functional/test_awsclient.py +++ b/tests/functional/test_awsclient.py @@ -1828,6 +1828,22 @@ def test_create_function_with_memory_size(self, stubbed_session): memory_size=256) == 'arn:12345:name' stubbed_session.verify_stubs() + def test_create_function_with_ephemeral_storage(self, stubbed_session): + stubbed_session.stub('lambda').create_function( + FunctionName='name', + Runtime='python2.7', + Code={'ZipFile': b'foo'}, + Handler='app.app', + Role='myarn', + EphemeralStorage={'Size': 1024}, + ).returns(self.SUCCESS_RESPONSE) + stubbed_session.activate_stubs() + awsclient = TypedAWSClient(stubbed_session) + assert awsclient.create_function( + 'name', 'myarn', b'foo', 'python2.7', 'app.app', + ephemeral_storage=1024) == 'arn:12345:name' + stubbed_session.verify_stubs() + def test_create_function_with_vpc_config(self, stubbed_session): stubbed_session.stub('lambda').create_function( FunctionName='name', @@ -2186,6 +2202,18 @@ def test_update_function_code_with_memory(self, stubbed_session): awsclient.update_function('name', b'foo', memory_size=256) stubbed_session.verify_stubs() + def test_update_function_code_with_ephemeral(self, stubbed_session): + lambda_client = stubbed_session.stub('lambda') + lambda_client.update_function_code( + FunctionName='name', ZipFile=b'foo').returns(self.SUCCESS_RESPONSE) + lambda_client.update_function_configuration( + FunctionName='name', + EphemeralStorage={'Size': 1024}).returns(self.SUCCESS_RESPONSE) + stubbed_session.activate_stubs() + awsclient = TypedAWSClient(stubbed_session) + awsclient.update_function('name', b'foo', ephemeral_storage=1024) + stubbed_session.verify_stubs() + def test_update_function_with_vpc_config(self, stubbed_session): lambda_client = stubbed_session.stub('lambda') lambda_client.update_function_code( diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index e109bb415..43eade6bb 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -241,6 +241,7 @@ def lambda_function(): tags={}, timeout=None, memory_size=None, + ephemeral_storage=None, role=models.PreCreatedIAMRole(role_arn='foobar'), security_group_ids=[], subnet_ids=[], diff --git a/tests/unit/deploy/test_appgraph.py b/tests/unit/deploy/test_appgraph.py index 0a2a023e0..2c5bcf2d1 100644 --- a/tests/unit/deploy/test_appgraph.py +++ b/tests/unit/deploy/test_appgraph.py @@ -126,6 +126,7 @@ def test_can_build_single_lambda_function_app(self, tags=config.tags, timeout=None, memory_size=None, + ephemeral_storage=None, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE), role=models.PreCreatedIAMRole('role:arn'), @@ -159,6 +160,7 @@ def test_can_build_single_lambda_function_app_with_log_retention( tags=config.tags, timeout=None, memory_size=None, + ephemeral_storage=None, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE), role=models.PreCreatedIAMRole('role:arn'), @@ -195,6 +197,7 @@ def test_can_build_single_lambda_function_app_with_managed_layer( tags=config.tags, timeout=None, memory_size=None, + ephemeral_storage=None, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE), role=models.PreCreatedIAMRole('role:arn'), @@ -251,6 +254,7 @@ def test_can_build_lambda_function_with_layers(self, tags=config.tags, timeout=None, memory_size=None, + ephemeral_storage=None, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE), role=models.PreCreatedIAMRole('role:arn'), @@ -310,6 +314,7 @@ def foo(event, context): tags=config.tags, timeout=None, memory_size=None, + ephemeral_storage=None, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE), role=models.PreCreatedIAMRole('role:arn'), @@ -372,6 +377,7 @@ def test_can_build_lambda_function_app_with_reserved_concurrency( tags=config.tags, timeout=None, memory_size=None, + ephemeral_storage=None, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE), role=models.PreCreatedIAMRole('role:arn'), diff --git a/tests/unit/deploy/test_deployer.py b/tests/unit/deploy/test_deployer.py index 4db630391..4965b3e2e 100644 --- a/tests/unit/deploy/test_deployer.py +++ b/tests/unit/deploy/test_deployer.py @@ -277,6 +277,7 @@ def create_function_resource(name): tags={}, timeout=60, memory_size=128, + ephemeral_storage=512, deployment_package=models.DeploymentPackage( models.Placeholder.BUILD_STAGE ), @@ -561,6 +562,7 @@ def test_inject_when_values_are_none(self): injector = InjectDefaults( lambda_timeout=100, lambda_memory_size=512, + lambda_ephemeral_storage=1024, ) function = models.LambdaFunction( # The timeout/memory_size are set to @@ -568,6 +570,7 @@ def test_inject_when_values_are_none(self): # in the with the default values above. timeout=None, memory_size=None, + ephemeral_storage=None, resource_name='foo', function_name='app-dev-foo', environment_variables={}, @@ -586,6 +589,7 @@ def test_inject_when_values_are_none(self): injector.handle(config, function) assert function.timeout == 100 assert function.memory_size == 512 + assert function.ephemeral_storage == 1024 def test_no_injection_when_values_are_set(self): injector = InjectDefaults( @@ -594,10 +598,11 @@ def test_no_injection_when_values_are_set(self): ) function = models.LambdaFunction( # The timeout/memory_size are set to - # None, so the injector should fill them + # 1, so the injector shouldn't fill them # in the with the default values above. timeout=1, memory_size=1, + ephemeral_storage=1, resource_name='foo', function_name='app-stage-foo', environment_variables={}, @@ -616,6 +621,7 @@ def test_no_injection_when_values_are_set(self): injector.handle(config, function) assert function.timeout == 1 assert function.memory_size == 1 + assert function.ephemeral_storage == 1 def test_default_tls_version_on_domain_name(self): injector = InjectDefaults(tls_version='TLS_1_2') diff --git a/tests/unit/deploy/test_planner.py b/tests/unit/deploy/test_planner.py index df02680d2..d46602a40 100644 --- a/tests/unit/deploy/test_planner.py +++ b/tests/unit/deploy/test_planner.py @@ -18,7 +18,7 @@ def create_function_resource(name, function_name=None, environment_variables=None, runtime='python2.7', handler='app.app', - tags=None, timeout=60, + tags=None, timeout=60, ephemeral_storage=512, memory_size=128, deployment_package=None, role=None, layers=None, managed_layer=None): if function_name is None: @@ -40,6 +40,7 @@ def create_function_resource(name, function_name=None, tags=tags, timeout=timeout, memory_size=memory_size, + ephemeral_storage=ephemeral_storage, xray=None, deployment_package=deployment_package, role=role, @@ -603,6 +604,7 @@ def test_can_create_function(self): 'xray': None, 'timeout': 60, 'memory_size': 128, + 'ephemeral_storage': 512, 'security_group_ids': [], 'subnet_ids': [], 'layers': [], @@ -655,6 +657,7 @@ def test_create_function_with_layers(self): 'timeout': 60, 'xray': None, 'memory_size': 128, + 'ephemeral_storage': 512, 'security_group_ids': [], 'subnet_ids': [], 'layers': [Variable('layer_version_arn')] + layers @@ -690,6 +693,7 @@ def test_can_update_lambda_function_code(self): 'xray': None, 'tags': {}, 'timeout': 60, + 'ephemeral_storage': 512, 'security_group_ids': [], 'subnet_ids': [], 'layers': [], @@ -755,6 +759,7 @@ def test_can_create_function_with_reserved_concurrency(self): 'xray': None, 'timeout': 60, 'memory_size': 128, + 'ephemeral_storage': 512, 'security_group_ids': [], 'subnet_ids': [], 'layers': [], diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 42e0eb8f8..b94f333e2 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -205,6 +205,7 @@ def test_can_create_scope_obj_with_new_function(): 'environment_variables': {'env': 'stage'}, 'lambda_timeout': 1, 'lambda_memory_size': 1, + 'lambda_ephemeral_storage': 1, 'tags': {'tag': 'stage'}, 'lambda_functions': { 'api_handler': { @@ -222,6 +223,7 @@ def test_can_create_scope_obj_with_new_function(): 'environment_variables': {'env': 'function'}, 'lambda_timeout': 2, 'lambda_memory_size': 2, + 'lambda_ephemeral_storage': 2, 'tags': {'tag': 'function'}, } } @@ -238,6 +240,7 @@ def test_can_create_scope_obj_with_new_function(): assert new_config.environment_variables == {'env': 'function'} assert new_config.lambda_timeout == 2 assert new_config.lambda_memory_size == 2 + assert new_config.lambda_ephemeral_storage == 2 assert new_config.tags['tag'] == 'function' @@ -454,6 +457,42 @@ def test_set_lambda_memory_size_override(self): assert c.lambda_memory_size == 256 +class TestConfigureLambdaEphemeralStorage(object): + def test_not_set(self): + c = Config('dev', config_from_disk={}) + assert c.lambda_ephemeral_storage is None + + def test_set_lambda_ephemeral_storage_global(self): + config_from_disk = { + 'lambda_ephemeral_storage': 1024 + } + c = Config('dev', config_from_disk=config_from_disk) + assert c.lambda_ephemeral_storage == 1024 + + def test_set_lambda_ephemeral_storage_stage(self): + config_from_disk = { + 'stages': { + 'dev': { + 'lambda_ephemeral_storage': 1024 + } + } + } + c = Config('dev', config_from_disk=config_from_disk) + assert c.lambda_ephemeral_storage == 1024 + + def test_set_lambda_ephemeral_storage_override(self): + config_from_disk = { + 'lambda_ephemeral_storage': 512, + 'stages': { + 'dev': { + 'lambda_ephemeral_storage': 1024 + } + } + } + c = Config('dev', config_from_disk=config_from_disk) + assert c.lambda_ephemeral_storage == 1024 + + class TestConfigureLambdaTimeout(object): def test_not_set(self): c = Config('dev', config_from_disk={}) diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py index cd01e451d..543a88f04 100644 --- a/tests/unit/test_package.py +++ b/tests/unit/test_package.py @@ -301,6 +301,7 @@ def lambda_function(self): timeout=120, xray=None, memory_size=128, + ephemeral_storage=512, deployment_package=models.DeploymentPackage(filename='foo.zip'), role=models.PreCreatedIAMRole(role_arn='role:arn'), security_group_ids=[], @@ -1158,6 +1159,7 @@ def test_sam_injects_policy(self, sample_app): tags={'foo': 'bar'}, timeout=120, memory_size=128, + ephemeral_storage=512, xray=None, deployment_package=models.DeploymentPackage(filename='foo.zip'), role=models.ManagedIAMRole( @@ -1179,6 +1181,7 @@ def test_sam_injects_policy(self, sample_app): 'CodeUri': 'foo.zip', 'Handler': 'app.app', 'MemorySize': 128, + 'EphemeralStorage': {'Size': 512}, 'Tracing': 'PassThrough', 'Role': {'Fn::GetAtt': ['Role', 'Arn']}, 'Runtime': 'python27', @@ -1260,6 +1263,7 @@ def test_role_arn_inserted_when_necessary(self): tags={'foo': 'bar'}, timeout=120, memory_size=128, + ephemeral_storage=512, xray=None, deployment_package=models.DeploymentPackage(filename='foo.zip'), role=models.PreCreatedIAMRole(role_arn='role:arn'), @@ -1276,6 +1280,7 @@ def test_role_arn_inserted_when_necessary(self): 'CodeUri': 'foo.zip', 'Handler': 'app.app', 'MemorySize': 128, + 'EphemeralStorage': {'Size': 512}, 'Role': 'role:arn', 'Tracing': 'PassThrough', 'Runtime': 'python27',