From d985ab9d678a035ad70035928011c5922159d587 Mon Sep 17 00:00:00 2001 From: Hannes Diedrich Date: Tue, 24 Nov 2020 10:27:29 +0100 Subject: [PATCH 1/3] deprecated_write_zip_results --- integration_tests/steps/integration_tests.py | 6 ----- .../redis_connections/redis_communication.py | 23 ------------------- 2 files changed, 29 deletions(-) diff --git a/integration_tests/steps/integration_tests.py b/integration_tests/steps/integration_tests.py index 283f52b28..23da216bc 100644 --- a/integration_tests/steps/integration_tests.py +++ b/integration_tests/steps/integration_tests.py @@ -548,12 +548,6 @@ def final_res_count(_): context.simulation.redis_connection.publish_results = final_res_count -@when('the simulation is able to transmit zipped results') -def transmit_zipped_results(context): - context.simulation.redis_connection.is_enabled = lambda: True - context.simulation.redis_connection.write_zip_results = lambda _: None - - @then('intermediate results are transmitted on every slot') def interm_res_report(context): # Add an extra result for the start of the simulation diff --git a/src/d3a/d3a_core/redis_connections/redis_communication.py b/src/d3a/d3a_core/redis_connections/redis_communication.py index 1cc1cd517..3be390180 100644 --- a/src/d3a/d3a_core/redis_connections/redis_communication.py +++ b/src/d3a/d3a_core/redis_connections/redis_communication.py @@ -202,29 +202,6 @@ def publish_results(self, endpoint_buffer): self.redis_db.publish(self.result_channel, results) self._handle_redis_job_metadata() - def write_zip_results(self, zip_results): - if not self.is_enabled(): - return - - message_size_kb = os.stat(zip_results).st_size / 1000.0 - if message_size_kb > 30000: - log.error(f"Do not publish simulation results bigger than 30 MB, current message " - f"size {message_size_kb / 1000.0} MB.") - return - - fp = open(zip_results, 'rb') - zip_data = fp.read() - fp.close() - - zip_results_key = ZIP_RESULTS_KEY + str(self._simulation_id) - - # Write results to a separate Redis key - self.redis_db.set(zip_results_key, zip_data) - # Inform d3a-web that a new zip file is available on this key - self.redis_db.publish(ZIP_RESULTS_CHANNEL, json.dumps( - {"job_id": self._simulation_id, "zip_redis_key": zip_results_key} - )) - def publish_intermediate_results(self, endpoint_buffer): # Should have a different format in the future, hence the code duplication self.publish_results(endpoint_buffer) From 1581662bf9e3048a5821efb66c129d27bb438262 Mon Sep 17 00:00:00 2001 From: Hannes Diedrich Date: Tue, 24 Nov 2020 12:08:11 +0100 Subject: [PATCH 2/3] fixed integration tests --- integration_tests/integration_tests.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/integration_tests.feature b/integration_tests/integration_tests.feature index 43bb69649..4919b8599 100644 --- a/integration_tests/integration_tests.feature +++ b/integration_tests/integration_tests.feature @@ -81,7 +81,6 @@ Feature: Run integration tests When a simulation is created for scenario default_2a And the simulation is able to transmit intermediate results And the simulation is able to transmit final results - And the simulation is able to transmit zipped results And the configured simulation is running Then intermediate results are transmitted on every slot And final results are transmitted once From ff9068088dc9abe683fc0159559939245a7549b2 Mon Sep 17 00:00:00 2001 From: Hannes Diedrich Date: Tue, 24 Nov 2020 15:47:37 +0100 Subject: [PATCH 3/3] fixed integration tests --- integration_tests/integration_tests.feature | 1 + integration_tests/steps/integration_tests.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/integration_tests/integration_tests.feature b/integration_tests/integration_tests.feature index 4919b8599..effa090c3 100644 --- a/integration_tests/integration_tests.feature +++ b/integration_tests/integration_tests.feature @@ -79,6 +79,7 @@ Feature: Run integration tests Scenario: Simulation publishes intermediate and final results Given d3a is installed When a simulation is created for scenario default_2a + When the redis_connection is enabled And the simulation is able to transmit intermediate results And the simulation is able to transmit final results And the configured simulation is running diff --git a/integration_tests/steps/integration_tests.py b/integration_tests/steps/integration_tests.py index 23da216bc..0a83da452 100644 --- a/integration_tests/steps/integration_tests.py +++ b/integration_tests/steps/integration_tests.py @@ -548,6 +548,11 @@ def final_res_count(_): context.simulation.redis_connection.publish_results = final_res_count +@when('the redis_connection is enabled') +def transmit_zipped_results(context): + context.simulation.redis_connection.is_enabled = lambda: True + + @then('intermediate results are transmitted on every slot') def interm_res_report(context): # Add an extra result for the start of the simulation