From 91be3e15a6e6b6ebc6929db16259bf61bab8c6e3 Mon Sep 17 00:00:00 2001 From: marrobi Date: Wed, 6 Dec 2023 10:44:08 +0000 Subject: [PATCH 1/3] Amend session ID to prevent parallel operations on a resource. --- api_app/service_bus/deployment_status_updater.py | 2 +- api_app/service_bus/helpers.py | 13 +++++++++++-- api_app/service_bus/resource_request_sender.py | 2 +- .../test_api/test_routes/test_resource_helpers.py | 2 +- templates/workspace_services/databricks/porter.yaml | 10 +++++++++- .../databricks/terraform/outputs.tf | 4 ++++ 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 8c09e83d77..44006b9266 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -163,7 +163,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # create + send the message logging.info(f"Sending next step in operation to deployment queue -> step_id: {next_step.templateStepId}, action: {next_step.resourceAction}") content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=next_step.id, action=next_step.resourceAction)) - await send_deployment_message(content=content, correlation_id=operation.id, session_id=resource_to_send.id, action=next_step.resourceAction) + await send_deployment_message(content=content, correlation_id=operation.id, resource=resource_to_send, action=next_step.resourceAction) except Exception as e: logging.exception("Unable to send update for resource in pipeline step") next_step.message = repr(e) diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index 729b9953af..e46ddd7fc5 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -34,7 +34,16 @@ async def _send_message(message: ServiceBusMessage, queue: str): await sender.send_messages(message) -async def send_deployment_message(content, correlation_id, session_id, action): +async def send_deployment_message(content, correlation_id, resource, action): + + # use session id to prevent likelihood of conflicts when sending multiple messages for the same resource + if resource.resourceType == ResourceType.WorkspaceService: + session_id = resource.workspaceId + elif resource.resourceType == ResourceType.UserResource: + session_id = resource.parentWorkspaceServiceId + else: + session_id = resource.id + resource_request_message = ServiceBusMessage(body=content, correlation_id=correlation_id, session_id=session_id) logging.info(f"Sending resource request message with correlation ID {resource_request_message.correlation_id}, action: {action}") await _send_message(resource_request_message, config.SERVICE_BUS_RESOURCE_REQUEST_QUEUE) @@ -42,7 +51,7 @@ async def send_deployment_message(content, correlation_id, session_id, action): async def update_resource_for_step(operation_step: OperationStep, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, root_resource: Resource, step_resource: Resource, resource_to_update_id: str, primary_action: str, user: User) -> Resource: # step_resource is the resource instance where the step was defined. e.g. 'add firewall rule' step defined in Guacamole template -> the step_resource is the Guacamole ws service. - # root_resource is theresource on which the user chose to update, i.e. the top most resource in cascaded action or the same resource in a non-cascaded action. + # root_resource is the resource on which the user chose to update, i.e. the top most resource in cascaded action or the same resource in a non-cascaded action. if step_resource is None: step_resource = await resource_repo.get_resource_by_id(operation_step.sourceTemplateResourceId) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index a0aac86aeb..8405f32228 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -55,6 +55,6 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope # create + send the message content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) - await send_deployment_message(content=content, correlation_id=operation.id, session_id=first_step.resourceId, action=first_step.resourceAction) + await send_deployment_message(content=content, correlation_id=operation.id, resource=resource_to_send, action=first_step.resourceAction) return operation diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index 092662930e..6aa8a2fd21 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -297,7 +297,7 @@ async def test_save_and_deploy_masks_secrets(self, send_deployment_message_mock, send_deployment_message_mock.assert_called_once_with( content=json.dumps(resource.get_resource_request_message_payload(operation_id=operation_id, step_id=step_id, action="install")), correlation_id=operation_id, - session_id=resource.id, + resource=resource, action="install") # Checking that the item saved had a secret redacted diff --git a/templates/workspace_services/databricks/porter.yaml b/templates/workspace_services/databricks/porter.yaml index d54a446c1f..d180cc1d9b 100644 --- a/templates/workspace_services/databricks/porter.yaml +++ b/templates/workspace_services/databricks/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-databricks -version: 1.0.3 +version: 1.0.8 description: "An Azure TRE service for Azure Databricks." registry: azuretre dockerfile: Dockerfile.tmpl @@ -29,6 +29,7 @@ parameters: type: string - name: is_exposed_externally type: boolean + default: false - name: tfstate_resource_group_name type: string description: "Resource group containing the Terraform state storage account" @@ -100,6 +101,11 @@ outputs: applyTo: - install - upgrade + - name: is_exposed_externally + type: boolean + applyTo: + - install + - upgrade mixins: - terraform: @@ -131,6 +137,7 @@ install: - name: artifact_blob_storage_domains - name: workspace_address_spaces - name: databricks_address_prefixes + - name: is_exposed_externally upgrade: - terraform: @@ -158,6 +165,7 @@ upgrade: - name: artifact_blob_storage_domains - name: workspace_address_spaces - name: databricks_address_prefixes + - name: is_exposed_externally uninstall: - terraform: diff --git a/templates/workspace_services/databricks/terraform/outputs.tf b/templates/workspace_services/databricks/terraform/outputs.tf index 800dd1f743..f814be441f 100644 --- a/templates/workspace_services/databricks/terraform/outputs.tf +++ b/templates/workspace_services/databricks/terraform/outputs.tf @@ -49,3 +49,7 @@ data "dns_a_record_set" "event_hub_endpoint_addresses" { output "event_hub_endpoint_addresses" { value = setunion(flatten([for addr in data.dns_a_record_set.event_hub_endpoint_addresses : addr.addrs])) } + +output "is_exposed_externally" { + value = var.is_exposed_externally +} From fceb03592d78ac5e73d508c8c5a35d71acbd1f04 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Fri, 8 Nov 2024 09:41:49 +0000 Subject: [PATCH 2/3] Update API version and changelog --- CHANGELOG.md | 3 ++- api_app/_version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4898f1f5ee..c857e8b148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ ENHANCEMENTS: * Update Terraform to use Azure AD authentication rather than storage account keys ([#4103](https://github.com/microsoft/AzureTRE/issues/4103)) BUG FIXES: -- Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) +* Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) +* Amend session ID to prevent parallel operations on a resource. ([#3807](https://github.com/microsoft/AzureTRE/pull/3807)) COMPONENTS: diff --git a/api_app/_version.py b/api_app/_version.py index 1a95d56216..8261536c68 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.19.3" +__version__ = "0.19.4" From cd8d1a86f8c50377ca1538b05400937a4d912cc1 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Fri, 8 Nov 2024 11:22:43 +0000 Subject: [PATCH 3/3] fix linting --- api_app/service_bus/resource_request_sender.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 91b5e4967e..c89651e1fd 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -64,5 +64,4 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) await send_deployment_message(content=content, correlation_id=operation.id, resource=resource_to_send, action=first_step.resourceAction) - return operation