Skip to content

Commit

Permalink
Merge branch 'master' into deleted-area-redis_external_connection.py
Browse files Browse the repository at this point in the history
  • Loading branch information
hannesdiedrich authored Nov 24, 2020
2 parents 987ac9e + a1c747c commit b21809a
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 26 deletions.
2 changes: 1 addition & 1 deletion integration_tests/integration_tests.feature
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ Feature: Run integration tests
Scenario: Simulation publishes intermediate and final results
Given d3a is installed
When a simulation is created for scenario default_2a
When the redis_connection is enabled
And the simulation is able to transmit intermediate results
And the simulation is able to transmit final results
And the simulation is able to transmit zipped results
And the configured simulation is running
Then intermediate results are transmitted on every slot
And final results are transmitted once
Expand Down
3 changes: 1 addition & 2 deletions integration_tests/steps/integration_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,10 +548,9 @@ def final_res_count(_):
context.simulation.redis_connection.publish_results = final_res_count


@when('the simulation is able to transmit zipped results')
@when('the redis_connection is enabled')
def transmit_zipped_results(context):
context.simulation.redis_connection.is_enabled = lambda: True
context.simulation.redis_connection.write_zip_results = lambda _: None


@then('intermediate results are transmitted on every slot')
Expand Down
23 changes: 0 additions & 23 deletions src/d3a/d3a_core/redis_connections/redis_communication.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,29 +202,6 @@ def publish_results(self, endpoint_buffer):
self.redis_db.publish(self.result_channel, results)
self._handle_redis_job_metadata()

def write_zip_results(self, zip_results):
if not self.is_enabled():
return

message_size_kb = os.stat(zip_results).st_size / 1000.0
if message_size_kb > 30000:
log.error(f"Do not publish simulation results bigger than 30 MB, current message "
f"size {message_size_kb / 1000.0} MB.")
return

fp = open(zip_results, 'rb')
zip_data = fp.read()
fp.close()

zip_results_key = ZIP_RESULTS_KEY + str(self._simulation_id)

# Write results to a separate Redis key
self.redis_db.set(zip_results_key, zip_data)
# Inform d3a-web that a new zip file is available on this key
self.redis_db.publish(ZIP_RESULTS_CHANNEL, json.dumps(
{"job_id": self._simulation_id, "zip_redis_key": zip_results_key}
))

def publish_intermediate_results(self, endpoint_buffer):
# Should have a different format in the future, hence the code duplication
self.publish_results(endpoint_buffer)
Expand Down

0 comments on commit b21809a

Please sign in to comment.