Merge branch 'master' into sentry-filtering-AnalysisError

cdci_data_analysis/analysis/instrument.py

@@ -35,6 +35,7 @@
 from urllib.parse import urlencode
 
 from cdci_data_analysis.analysis.queries import _check_is_base_query
+from .parameters import POSIXPath
 from ..analysis import tokenHelper, parameters
 from .catalog import BasicCatalog
 from .products import QueryOutput
@@ -167,14 +168,16 @@ def set_pars_from_dic(self, arg_dic, verbose=False):
         for par in param_list:
             self.logger.info("before normalizing, set_pars_from_dic>> par: %s par.name: %s par.value: %s par_dic[par.name]: %s",
                              par, par.name, par.value, arg_dic.get(par.name, None))
-
             # this is required because in some cases a parameter is set without a name (eg UserCatalog),
             # or they don't have to set (eg scw_list)
             if par.name is not None and par.name not in params_not_to_be_included:
+                if isinstance(par, POSIXPath) and par.name + '_type' in arg_dic and arg_dic[par.name + '_type'] == 'file'\
+                        and par.name not in arg_dic:
+                    par.value = None
+
                 # set the value for par to a default format,
                 # or to a default value if this is not included within the request
                 updated_arg_dic[par.name] = par.set_value_from_form(arg_dic, verbose=verbose)
-
                 if par.units_name is not None:
                     if par.default_units is not None:
                         updated_arg_dic[par.units_name] = par.default_units

cdci_data_analysis/plugins/dummy_plugin/dummy_plugin_conf.yml

@@ -5,3 +5,4 @@ instruments:
 - empty_development_instrument
 - empty_async_return_progress_instrument
 - empty_instrument_with_conf
+- empty_instrument_with_posix_path

cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py

@@ -0,0 +1,64 @@
+"""
+Overview
+--------
+   
+general info about this module
+
+
+Classes and Inheritance Structure
+----------------------------------------------
+.. inheritance-diagram:: 
+
+Summary
+---------
+.. autosummary::
+   list of the module you want
+    
+Module API
+----------
+"""
+
+from __future__ import absolute_import, division, print_function
+
+__author__ = "Andrea Tramacere"
+
+# Standard library
+# eg copy
+# absolute import rg:from copy import deepcopy
+
+# Dependencies
+# eg numpy 
+# absolute import eg: import numpy as np
+
+# Project
+# relative import eg: from .mod import f
+
+
+from cdci_data_analysis.analysis.instrument import Instrument
+from cdci_data_analysis.analysis.queries import SourceQuery, InstrumentQuery
+
+from .data_server_dispatcher import FileParameterQuery
+
+from ...analysis.parameters import POSIXPath
+from .empty_instrument import BoundaryFloat
+
+def my_instr_factory():
+    src_query = SourceQuery('src_query')
+
+    instr_query = InstrumentQuery(name='empty_instrument_query',
+                                  input_prod_list_name='p_list',
+                                  catalog=None,
+                                  catalog_name='user_catalog')
+
+    f = POSIXPath(value=None, name='dummy_POSIX_file', is_optional=True)
+    p = BoundaryFloat(value=10., name='p', units='W', )
+    file_query = FileParameterQuery('file_parameters_dummy_query',
+                                    parameters_list=[f, p])
+
+    query_dictionary = {'file_dummy': 'file_parameters_dummy_query'}
+
+    return Instrument('empty-with-posix-path',
+                      src_query=src_query,
+                      instrumet_query=instr_query,
+                      product_queries_list=[file_query,],
+                      query_dictionary=query_dictionary)

cdci_data_analysis/plugins/dummy_plugin/exposer.py

@@ -4,6 +4,7 @@
 from . import empty_development_instrument
 from . import empty_async_return_progress_instrument
 from . import empty_instrument_with_conf
+from . import empty_instrument_with_posix_path
 from . import conf_file
 import yaml
 

tests/test_server_basic.py

@@ -161,7 +161,7 @@ def test_empty_request(dispatcher_live_fixture):
     assert c.status_code == 400
 
     # parameterize this
-    assert sorted(jdata['installed_instruments']) == sorted(['empty', 'empty-async', 'empty-with-conf', 'empty-semi-async', 'empty-development', 'empty-async-return-progress']) or \
+    assert sorted(jdata['installed_instruments']) == sorted(['empty', 'empty-async', 'empty-with-conf', 'empty-semi-async', 'empty-development', 'empty-async-return-progress', 'empty-with-posix-path']) or \
            jdata['installed_instruments'] == []
 
     assert jdata['debug_mode'] == "yes"
@@ -255,7 +255,7 @@ def test_matrix_options_mode_empty_request(dispatcher_live_fixture_with_matrix_o
     assert c.status_code == 400
 
     assert sorted(jdata['installed_instruments']) == sorted(
-        ['empty', 'empty-async', 'empty-semi-async', 'empty-with-conf', 'empty-development', 'empty-async-return-progress']) or \
+        ['empty', 'empty-async', 'empty-semi-async', 'empty-with-conf', 'empty-development', 'empty-async-return-progress', 'empty-with-posix-path',]) or \
            jdata['installed_instruments'] == []
 
     # assert jdata['debug_mode'] == "no"
@@ -1988,6 +1988,58 @@ def test_public_file_ownerships(dispatcher_live_fixture):
     assert ownerships['user_roles'] == []
 
 
+@pytest.mark.parametrize("include_file_arg", [True, False])
+def test_default_value_empty_posix_path(dispatcher_live_fixture, include_file_arg):
+    DispatcherJobState.remove_scratch_folders()
+    DispatcherJobState.empty_request_files_folders()
+    server = dispatcher_live_fixture
+    logger.info("constructed server: %s", server)
+
+    # let's generate a valid token
+    token_payload = {
+        **default_token_payload,
+        "roles": "unige-hpc-full, general",
+    }
+    encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256')
+
+    params = {
+        **default_params,
+        'product_type': 'file_dummy',
+        'query_type': "Dummy",
+        'instrument': 'empty-with-posix-path',
+        'dummy_POSIX_file_type': 'file',
+        'token': encoded_token
+    }
+
+    p_file_path = DispatcherJobState.create_p_value_file(p_value=6)
+    list_file = open(p_file_path)
+
+    expected_query_status = 'done'
+    expected_job_status = 'done'
+    expected_status_code = 200
+
+    files = None
+    if include_file_arg:
+        files = {'dummy_POSIX_file': list_file.read()}
+
+    jdata = ask(server,
+                params,
+                expected_query_status=expected_query_status,
+                expected_job_status=expected_job_status,
+                expected_status_code=expected_status_code,
+                max_time_s=150,
+                method='post',
+                files=files
+                )
+
+    list_file.close()
+    assert 'dummy_POSIX_file' in jdata['products']['analysis_parameters']
+    if include_file_arg:
+        assert jdata['products']['analysis_parameters']['dummy_POSIX_file'] is not None
+    else:
+        assert jdata['products']['analysis_parameters']['dummy_POSIX_file'] is None
+
+
 def test_scws_list_file(dispatcher_live_fixture):
 
     server = dispatcher_live_fixture