diff --git a/scripts/run_all_tests.bat b/scripts/run_all_tests.bat new file mode 100644 index 0000000..44df215 --- /dev/null +++ b/scripts/run_all_tests.bat @@ -0,0 +1,2 @@ +@echo off +call %CD%\..\..\..\scripts\generic_test_script.bat sostrades_optimization_plugins diff --git a/sostrades_optimization_plugins/models/func_manager/func_manager.py b/sostrades_optimization_plugins/models/func_manager/func_manager.py index 68960ae..64198a9 100644 --- a/sostrades_optimization_plugins/models/func_manager/func_manager.py +++ b/sostrades_optimization_plugins/models/func_manager/func_manager.py @@ -18,6 +18,10 @@ from sostrades_core.tools.base_functions.exp_min import compute_func_with_exp_min from sostrades_optimization_plugins.tools.cst_manager.func_manager_common import ( + keep_negative_only, + keep_negative_only_square, + keep_positive_only, + keep_positive_only_square, smooth_maximum, ) @@ -35,10 +39,17 @@ class FunctionManager: WEIGHT = 'weight' # Can be used for normalisation AGGR = 'aggr' AGGR_TYPE_SMAX = 'smax' + INEQ_NEGATIVE_WHEN_SATIFIED = 'negative_when_satisfied' + INEQ_POSITIVE_WHEN_SATIFIED = 'positive_when_satisfied' + INEQ_NEGATIVE_WHEN_SATIFIED_AND_SQUARE_IT = 'negative_when_satisfied_square_it' + INEQ_POSITIVE_WHEN_SATIFIED_AND_SQUARE_IT = 'positive_when_satisfied_square_it' AGGR_TYPE_SUM = 'sum' + AGGR_TYPE_MEAN = 'mean' AGGR_TYPE_DELTA = 'delta' AGGR_TYPE_LIN_TO_QUAD = 'lin_to_quad' - POS_AGGR_TYPE = [AGGR_TYPE_SMAX, AGGR_TYPE_SUM, AGGR_TYPE_DELTA, AGGR_TYPE_LIN_TO_QUAD] + POS_AGGR_TYPE = [AGGR_TYPE_SMAX, AGGR_TYPE_SUM, AGGR_TYPE_DELTA, AGGR_TYPE_LIN_TO_QUAD, + AGGR_TYPE_MEAN, INEQ_NEGATIVE_WHEN_SATIFIED, INEQ_POSITIVE_WHEN_SATIFIED, + INEQ_NEGATIVE_WHEN_SATIFIED_AND_SQUARE_IT, INEQ_POSITIVE_WHEN_SATIFIED_AND_SQUARE_IT] def __init__(self): """ @@ -125,20 +136,31 @@ def scalarize_all_functions(self, eps=1e-3, alpha=3): if self.functions[tag][self.FTYPE] == self.OBJECTIVE: #-- smooth maximum of values return the value if it was a float #-- return smooth maximum if objective was an array - if aggr_type == 'smax': + if aggr_type == self.AGGR_TYPE_SMAX: res = smooth_maximum(values, alpha) - elif aggr_type == 'sum': + elif aggr_type == self.AGGR_TYPE_SUM: res = values.sum() + elif aggr_type == self.AGGR_TYPE_MEAN: + res = values.mean() else: raise Exception(f"Unhandled aggr_type {aggr_type}") elif self.functions[tag][self.FTYPE] == self.INEQ_CONSTRAINT: #-- scale between (0., +inf) and take smooth maximum - cst = self.cst_func_ineq(values, eps, tag) - res = smooth_maximum(cst, alpha) + if aggr_type == self.INEQ_NEGATIVE_WHEN_SATIFIED: + res = keep_positive_only(values) + elif aggr_type == self.INEQ_POSITIVE_WHEN_SATIFIED: + res = keep_negative_only(values) + elif aggr_type == self.INEQ_NEGATIVE_WHEN_SATIFIED_AND_SQUARE_IT: + res = keep_positive_only_square(values) + elif aggr_type == self.INEQ_POSITIVE_WHEN_SATIFIED_AND_SQUARE_IT: + res = keep_negative_only_square(values) + else: + cst = self.cst_func_ineq(values, eps, tag) + res = smooth_maximum(cst, alpha) elif self.functions[tag][self.FTYPE] == self.EQ_CONSTRAINT: - if aggr_type == 'delta': + if aggr_type == self.AGGR_TYPE_DELTA: cst = self.cst_func_eq_delta(values, eps, tag) - elif aggr_type == 'lin_to_quad': + elif aggr_type == self.AGGR_TYPE_LIN_TO_QUAD: cst = self.cst_func_eq_lintoquad(values, eps, tag) else: cst = self.cst_func_eq(values) diff --git a/sostrades_optimization_plugins/models/func_manager/func_manager_disc.py b/sostrades_optimization_plugins/models/func_manager/func_manager_disc.py index 54a3aa1..27b210b 100644 --- a/sostrades_optimization_plugins/models/func_manager/func_manager_disc.py +++ b/sostrades_optimization_plugins/models/func_manager/func_manager_disc.py @@ -39,7 +39,13 @@ FunctionManager, ) from sostrades_optimization_plugins.tools.cst_manager.func_manager_common import ( + derivative_d_keep_negative_only, + derivative_d_keep_positive_only, + derivative_d_keep_positive_only_square, get_dsmooth_dvariable, + keep_negative_only, + keep_positive_only, + keep_positive_only_square, smooth_maximum, ) @@ -85,6 +91,9 @@ class FunctionManagerDisc(OptimManagerDisc): OPTIM_OUTPUT_DF = 'optim_output_df' EXPORT_CSV = 'export_csv' + AGGR_TYPE_SUM = FunctionManager.AGGR_TYPE_SUM + AGGR_TYPE_SMAX = FunctionManager.AGGR_TYPE_SMAX + NS_OPTIM = 'ns_optim' DESC_IN = {FUNC_DF: {'type': 'dataframe', 'dataframe_descriptor': {VARIABLE: ('string', None, True), # input function @@ -102,8 +111,8 @@ class FunctionManagerDisc(OptimManagerDisc): EXPORT_CSV: {'type': 'bool', 'default': False}, 'smooth_log': {'type': 'bool', 'default': False, 'user_level': 3}, 'eps2': {'type': 'float', 'default': 1e10, 'user_level': 3}, - 'aggr_mod_ineq': {'type': 'string', 'default': 'sum', 'user_level': 3}, - 'aggr_mod_eq': {'type': 'string', 'default': 'sum', 'user_level': 3}, + 'aggr_mod_ineq': {'type': 'string', 'default': AGGR_TYPE_SUM, 'user_level': 3}, + 'aggr_mod_eq': {'type': 'string', 'default': AGGR_TYPE_SUM, 'user_level': 3}, } DESC_OUT = {OPTIM_OUTPUT_DF: {'type': 'dataframe'}} @@ -223,12 +232,12 @@ def setup_sos_disciplines(self): ftype = metadata[FunctionManager.FTYPE] w = metadata.get(FunctionManager.WEIGHT, None) - aggr_type = 'sum' + aggr_type = self.AGGR_TYPE_SUM if self.AGGR_TYPE in func_df.columns: aggr_type = func_df.loc[func_df[self.VARIABLE] == f, self.AGGR_TYPE].values[0] if pd.isnull(aggr_type): - aggr_type = 'sum' + aggr_type = self.AGGR_TYPE_SUM if w is None: w = 1. self.func_manager.add_function( @@ -274,7 +283,9 @@ def run(self): for f in self.function_dict.keys(): fvalue_df = self.get_sosdisc_inputs(f) self.check_isnan_inf(f, fvalue_df) - self.check_value_range(fvalue_df, fname=f, ftype=self.function_dict[f]['ftype'].upper()) + weight = self.function_dict[f][self.WEIGHT] + if weight != 0: + self.check_value_range(fvalue_df, fname=f, ftype=self.function_dict[f]['ftype'].upper()) # conversion dataframe > array: f_arr = self.convert_df_to_array(f, fvalue_df) # update func manager with value as array @@ -407,20 +418,22 @@ def compute_sos_jacobian(self): value_gh_l = [] value_ghk_l = [] for variable_name in self.func_manager.functions.keys(): - value_df = inputs_dict[variable_name] - weight = self.func_manager.functions[variable_name][self.WEIGHT] + input_var_value = inputs_dict[variable_name] grad_value_l[variable_name] = {} grad_val_compos[variable_name] = {} - if self.func_manager.functions[variable_name][self.FTYPE] == self.OBJECTIVE: + var_f_type = self.func_manager.functions[variable_name][self.FTYPE] + var_aggr_type = self.func_manager.functions[variable_name][self.AGGR_TYPE] + weight = self.func_manager.functions[variable_name][self.WEIGHT] + if var_f_type == self.OBJECTIVE: - if isinstance(value_df, np.ndarray): + if isinstance(input_var_value, np.ndarray): n = len( self.func_manager.functions[variable_name][self.VALUE]) - if self.func_manager.functions[variable_name][self.AGGR_TYPE] == 'sum': + if self.func_manager.functions[variable_name][self.AGGR_TYPE] == self.AGGR_TYPE_SUM: grad_value = weight * np.ones(n) - elif self.func_manager.functions[variable_name][self.AGGR_TYPE] == 'smax': + elif self.func_manager.functions[variable_name][self.AGGR_TYPE] == self.AGGR_TYPE_SMAX: grad_value = float(weight) * \ np.array(get_dsmooth_dvariable( self.func_manager.functions[variable_name][self.VALUE])) @@ -432,15 +445,15 @@ def compute_sos_jacobian(self): self.set_partial_derivative( 'objective', variable_name, np.atleast_2d(grad_value)) else: - for col_name in value_df.columns: + for col_name in input_var_value.columns: if col_name != 'years': n = len( self.func_manager.functions[variable_name][self.VALUE]) - if self.func_manager.functions[variable_name][self.AGGR_TYPE] == 'sum': + if self.func_manager.functions[variable_name][self.AGGR_TYPE] == self.AGGR_TYPE_SUM: grad_value = weight * np.ones(n) - elif self.func_manager.functions[variable_name][self.AGGR_TYPE] == 'smax': + elif self.func_manager.functions[variable_name][self.AGGR_TYPE] == self.AGGR_TYPE_SMAX: grad_value = float(weight) * \ np.array(get_dsmooth_dvariable( @@ -452,36 +465,65 @@ def compute_sos_jacobian(self): self.set_partial_derivative_for_other_types( ('objective_lagrangian',), (variable_name, col_name), 100.0 * grad_value) - elif self.func_manager.functions[variable_name][self.FTYPE] == self.INEQ_CONSTRAINT: - if isinstance(value_df, pd.DataFrame): - for col_name in value_df.columns: - if col_name != 'years': - # h: cst_func_ineq, g: smooth_max, f: smooth_max - # g(h(x)) for each variable , f([g(h(x1), g(h(x2))]) - # weight - weight = self.func_manager.functions[variable_name][self.WEIGHT] - value = value_df[col_name] * weight - # h'(x) - grad_value = self.get_dfunc_ineq_dvariable( - value) - # h(x) - func_ineq = f_manager.cst_func_ineq( - value) - # g(h(x)) - value_gh_l.append( - smooth_maximum(func_ineq)) - # g'(h(x)) - grad_val_compos[variable_name][col_name] = get_dsmooth_dvariable( - func_ineq) - # g'(h(x)) * h'(x) - grad_val_compos_l = np.array( - grad_value) * np.array(grad_val_compos[variable_name][col_name]) - - grad_value_l[variable_name][col_name] = grad_val_compos_l - - elif isinstance(value_df, np.ndarray): - weight = self.func_manager.functions[variable_name][self.WEIGHT] - value = value_df * weight + elif var_f_type == self.INEQ_CONSTRAINT: + if isinstance(input_var_value, pd.DataFrame): + if var_aggr_type == FunctionManager.INEQ_NEGATIVE_WHEN_SATIFIED: + for col_name in input_var_value.columns: + if col_name != 'years': + value = input_var_value[col_name].values * weight + func_value = keep_positive_only(value) + grad_value = derivative_d_keep_positive_only(value) + value_gh_l.append(func_value) + grad_value_l[variable_name][col_name] = grad_value + elif var_aggr_type == FunctionManager.INEQ_POSITIVE_WHEN_SATIFIED: + for col_name in input_var_value.columns: + if col_name != 'years': + value = input_var_value[col_name] * weight + func_value = keep_negative_only(value) + grad_value = derivative_d_keep_negative_only(value) + value_gh_l.append(func_value) + grad_value_l[variable_name][col_name] = grad_value + elif var_aggr_type == FunctionManager.INEQ_NEGATIVE_WHEN_SATIFIED_AND_SQUARE_IT: + for col_name in input_var_value.columns: + if col_name != 'years': + value = input_var_value[col_name] * weight + func_value = keep_positive_only_square(value) + grad_value = derivative_d_keep_positive_only_square(value) + value_gh_l.append(func_value) + grad_value_l[variable_name][col_name] = grad_value + elif var_aggr_type == FunctionManager.INEQ_POSITIVE_WHEN_SATIFIED_AND_SQUARE_IT: + for col_name in input_var_value.columns: + if col_name != 'years': + value = input_var_value[col_name] * weight + func_value = keep_negative_only(value) ** 2 + grad_value = derivative_d_keep_negative_only(value) * 2 * value + value_gh_l.append(func_value) + grad_value_l[variable_name][col_name] = grad_value + else: + for col_name in input_var_value.columns: + if col_name != 'years': + # h: cst_func_ineq, g: smooth_max, f: smooth_max + # g(h(x)) for each variable , f([g(h(x1), g(h(x2))]) + # weight + value = input_var_value[col_name] * weight + # h'(x) + grad_value = self.get_dfunc_ineq_dvariable(value) + # h(x) + func_ineq = f_manager.cst_func_ineq(value) + # g(h(x)) + value_gh_l.append( + smooth_maximum(func_ineq)) + # g'(h(x)) + grad_val_compos[variable_name][col_name] = get_dsmooth_dvariable( + func_ineq) + # g'(h(x)) * h'(x) + grad_val_compos_l = np.array( + grad_value) * np.array(grad_val_compos[variable_name][col_name]) + + grad_value_l[variable_name][col_name] = grad_val_compos_l + + elif isinstance(input_var_value, np.ndarray): + value = input_var_value * weight # h'(x) grad_value = self.get_dfunc_ineq_dvariable( value) @@ -502,15 +544,14 @@ def compute_sos_jacobian(self): else: raise Exception( 'Gradients for constraints which are not dataframes or arrays are not yet implemented') - elif self.func_manager.functions[variable_name][self.FTYPE] == self.EQ_CONSTRAINT: - if isinstance(value_df, pd.DataFrame): - for col_name in value_df.columns: + elif var_f_type == self.EQ_CONSTRAINT: + if isinstance(input_var_value, pd.DataFrame): + for col_name in input_var_value.columns: if col_name != 'years': # k: sqrt(x**2) h: cst_func_eq, g : smooth_max, f: smooth_max # g(h(k(x))) for each variable , f([g(h(k(x1)), g(h(k(x2)))]) # weight - weight = self.func_manager.functions[variable_name][self.WEIGHT] - value = np.array(value_df[col_name].values * weight) + value = np.array(input_var_value[col_name].values * weight) if self.func_manager.functions[variable_name][ self.AGGR_TYPE] == self.func_manager.AGGR_TYPE_DELTA: # k(x) @@ -559,12 +600,11 @@ def compute_sos_jacobian(self): grad_value_l[variable_name][col_name] = grad_val_compos_l - elif isinstance(value_df, np.ndarray): + elif isinstance(input_var_value, np.ndarray): # k: sqrt(x**2) h: cst_func_eq, g : smooth_max, f: smooth_max # g(h(k(x))) for each variable , f([g(h(k(x1)), g(h(k(x2)))]) # weight - weight = self.func_manager.functions[variable_name][self.WEIGHT] - value = value_df * weight + value = input_var_value * weight if self.func_manager.functions[variable_name][ self.AGGR_TYPE] == self.func_manager.AGGR_TYPE_DELTA: @@ -629,12 +669,13 @@ def compute_sos_jacobian(self): i = 0 j = 0 for variable_name in self.func_manager.functions.keys(): - if self.func_manager.functions[variable_name][self.FTYPE] == self.INEQ_CONSTRAINT: + weight = self.func_manager.functions[variable_name][self.WEIGHT] + var_f_type = self.func_manager.functions[variable_name][self.FTYPE] + if var_f_type == self.INEQ_CONSTRAINT: - value_df = inputs_dict[variable_name] + input_var_value = inputs_dict[variable_name] dict_grad_ineq[variable_name] = grad_val_ineq[i] - weight = self.func_manager.functions[variable_name][self.WEIGHT] - if isinstance(value_df, np.ndarray): + if isinstance(input_var_value, np.ndarray): if self.func_manager.aggr_mod_ineq == 'smooth_max': grad_lagr_val = np.array(grad_value_l[variable_name]) * grad_val_ineq[i] @@ -652,9 +693,8 @@ def compute_sos_jacobian(self): i = i + 1 - elif isinstance(value_df, pd.DataFrame): - - for col_name in value_df.columns: + elif isinstance(input_var_value, pd.DataFrame): + for col_name in input_var_value.columns: if col_name != 'years': if self.func_manager.aggr_mod_ineq == 'smooth_max': grad_lagr_val = np.array(grad_value_l[variable_name][col_name]) * grad_val_ineq[i] @@ -664,18 +704,18 @@ def compute_sos_jacobian(self): grad_ineq_val = np.array(grad_value_l[variable_name][col_name]) self.set_partial_derivative_for_other_types( - ('objective_lagrangian',), (variable_name, col_name), + ('objective_lagrangian',), + (variable_name, col_name), weight * 100.0 * grad_lagr_val) self.set_partial_derivative_for_other_types( ('ineq_constraint',), (variable_name, col_name), weight * grad_ineq_val) i = i + 1 - elif self.func_manager.functions[variable_name][self.FTYPE] == self.EQ_CONSTRAINT: - value_df = inputs_dict[variable_name] + elif var_f_type == self.EQ_CONSTRAINT: + input_var_value = inputs_dict[variable_name] dict_grad_eq[variable_name] = grad_val_eq[j] - weight = self.func_manager.functions[variable_name][self.WEIGHT] - if isinstance(value_df, np.ndarray): + if isinstance(input_var_value, np.ndarray): if self.func_manager.aggr_mod_eq == 'smooth_max': grad_lagr_val = np.array(grad_value_l[variable_name]) * grad_val_eq[j] grad_eq_val = np.array(grad_value_l[variable_name]) * grad_val_eq[j] @@ -692,9 +732,9 @@ def compute_sos_jacobian(self): np.atleast_2d(weight * grad_eq_val)) j = j + 1 - elif isinstance(value_df, pd.DataFrame): + elif isinstance(input_var_value, pd.DataFrame): - for col_name in value_df.columns: + for col_name in input_var_value.columns: if col_name != 'years': if self.func_manager.aggr_mod_eq == 'smooth_max': grad_lagr_val = np.array(grad_value_l[variable_name][col_name]) * grad_val_eq[j] @@ -909,7 +949,7 @@ def get_parameters_df(self, func_df): ineq_list = [] eq_list = [] mod_columns_dict = {self.PARENT: None, - self.WEIGHT: 1.0, self.AGGR_TYPE: 'sum'} + self.WEIGHT: 1.0, self.AGGR_TYPE: self.AGGR_TYPE_SUM} for i, row in func_df.iterrows(): mod_parameter_dict = {} mod_parameter_dict[self.VARIABLE] = row[self.VARIABLE] + '_mod' @@ -937,20 +977,20 @@ def get_parameters_df(self, func_df): # Aggregated objectives and constraints dict_aggr_obj = {self.VARIABLE: self.OBJECTIVE, self.PARENT: self.OBJECTIVE_LAGR, - self.WEIGHT: 1.0, self.AGGR_TYPE: 'sum'} + self.WEIGHT: 1.0, self.AGGR_TYPE: self.AGGR_TYPE_SUM} parameters_dict[dict_aggr_obj[self.VARIABLE]] = dict_aggr_obj dict_aggr_ineq = {self.VARIABLE: self.INEQ_CONSTRAINT, self.PARENT: self.OBJECTIVE_LAGR, - self.WEIGHT: 1.0, self.AGGR_TYPE: 'smax'} + self.WEIGHT: 1.0, self.AGGR_TYPE: self.AGGR_TYPE_SMAX} parameters_dict[dict_aggr_ineq[self.VARIABLE]] = dict_aggr_ineq dict_aggr_eq = {self.VARIABLE: self.EQ_CONSTRAINT, self.PARENT: self.OBJECTIVE_LAGR, - self.WEIGHT: 1.0, self.AGGR_TYPE: 'smax'} + self.WEIGHT: 1.0, self.AGGR_TYPE: self.AGGR_TYPE_SMAX} parameters_dict[dict_aggr_eq[self.VARIABLE]] = dict_aggr_eq # Lagrangian objective dict_lagrangian = {self.VARIABLE: self.OBJECTIVE_LAGR, self.PARENT: None, - self.WEIGHT: 1.0, self.AGGR_TYPE: 'sum'} + self.WEIGHT: 1.0, self.AGGR_TYPE: self.AGGR_TYPE_SUM} parameters_dict[dict_lagrangian[self.VARIABLE]] = dict_lagrangian parameters_df = pd.DataFrame(parameters_dict).transpose() @@ -1257,7 +1297,7 @@ def get_chart_parameters_mod_iterations(self, optim_output, parameters_df, name) *parameters_df.loc[children_list, 'value'])] dict_parent = {self.VARIABLE: parent, self.PARENT: parent_parent, - self.WEIGHT: 1.0, self.AGGR_TYPE: 'sum', + self.WEIGHT: 1.0, self.AGGR_TYPE: self.AGGR_TYPE_SUM, 'value': [value]} parameters_df = pd.concat([parameters_df, pd.DataFrame(dict_parent, index=[parent])], axis=0) @@ -1486,7 +1526,7 @@ def __build_mod_names(self, f): def check_value_range(self, fvalue_df, fname: str, ftype: str): """Log some warnings if value is not between 0 and 1""" if isinstance(fvalue_df, np.ndarray): - if fvalue_df.max() > 1: - self.logger.warning(f"{ftype} {fname} maximum is above 1 ({fvalue_df.max()}). All its values should be between 0 and 1") - if fvalue_df.min() < 0: - self.logger.warning(f"{ftype} {fname} minimum is lower than 1 ({fvalue_df.min()}). All its values should be between 0 and 1") + if fvalue_df.max() > 10: + self.logger.warning(f"{ftype} {fname} maximum is above 10 ({fvalue_df.max()}). Ideally all its values should be between 0 and 1") + if fvalue_df.min() < -10: + self.logger.warning(f"{ftype} {fname} minimum is lower than -10 ({fvalue_df.min()}). Ideally all its values should be between 0 and 1") diff --git a/sostrades_optimization_plugins/tests/l0_test_44_func_manager.py b/sostrades_optimization_plugins/tests/l0_test_44_func_manager.py index 8f79c7d..e9ef811 100644 --- a/sostrades_optimization_plugins/tests/l0_test_44_func_manager.py +++ b/sostrades_optimization_plugins/tests/l0_test_44_func_manager.py @@ -273,6 +273,8 @@ def test_07_jacobian_func_manager_disc(self): cst3 = base_df.copy() cst3['cst3_values'] = np.array([-10., 0.2, -5.]) + cst0 = base_df.copy() + cst0['cst0_values'] = np.array([-10., 0.2, -5.]) eqcst1 = base_df.copy() eqcst1['eqcst1_values'] = np.array([-10., 10., -5.]) eqcst2 = base_df.copy() @@ -281,12 +283,12 @@ def test_07_jacobian_func_manager_disc(self): # -- ~GUI inputs: selection of functions func_df = pd.DataFrame(columns=['variable', 'ftype', 'weight']) - func_df['variable'] = ['cst1', 'cst2', 'cst3', + func_df['variable'] = ['cst0','cst1', 'cst2', 'cst3', 'eqcst1', 'eqcst2', 'obj1', 'obj2'] - func_df['ftype'] = [INEQ_CONSTRAINT, INEQ_CONSTRAINT, + func_df['ftype'] = [INEQ_CONSTRAINT, INEQ_CONSTRAINT, INEQ_CONSTRAINT, INEQ_CONSTRAINT, EQ_CONSTRAINT, EQ_CONSTRAINT, OBJECTIVE, OBJECTIVE] - func_df['weight'] = [1., 1., 1., 1, 1, 0.8, 0.2] - func_df['aggr'] = "sum" + func_df['weight'] = [1, 1., 1., 1., 1, 1, 0.8, 0.2] + func_df['aggr'] = [FunctionManager.INEQ_NEGATIVE_WHEN_SATIFIED] + [FunctionManager.AGGR_TYPE_SUM] * 7 func_df['parent'] = 'obj' func_df['namespace'] = '' @@ -298,6 +300,7 @@ def test_07_jacobian_func_manager_disc(self): values_dict[prefix + 'cst1'] = cst1 values_dict[prefix + 'cst2'] = cst2 values_dict[prefix + 'cst3'] = cst3 + values_dict[prefix + 'cst0'] = cst0 values_dict[prefix + 'eqcst1'] = eqcst1 values_dict[prefix + 'eqcst2'] = eqcst2 values_dict[prefix + 'obj1'] = obj1 @@ -331,7 +334,9 @@ def test_07_jacobian_func_manager_disc(self): assert disc_techno.check_jacobian( input_data=disc_techno.local_data, - threshold=1e-5, inputs=['FuncManagerTest.FunctionManager.cst1', 'FuncManagerTest.FunctionManager.cst2', + threshold=1e-5, inputs=[ + 'FuncManagerTest.FunctionManager.cst0', + 'FuncManagerTest.FunctionManager.cst1', 'FuncManagerTest.FunctionManager.cst2', 'FuncManagerTest.FunctionManager.cst3', 'FuncManagerTest.FunctionManager.obj1', 'FuncManagerTest.FunctionManager.obj2'], outputs=['FuncManagerTest.FunctionManager.objective_lagrangian'], derr_approx='complex_step') @@ -609,7 +614,7 @@ def test_11_jacobian_eq_delta_and_lin_to_quad(self): obj2['obj2_values'] = 1. ineq_cst = base_df.copy() ineq_cst['ineq_cst_values'] = np.array([10., -2000., -30.]) - ineq_cst_array = np.array([10., 2000., -30.]) + ineq_cst0 = np.array([10., 2000., -30.]) eqcst_delta = base_df.copy() eqcst_delta['eqcst_delta_values'] = np.array([400., 1., -10.]) eqcst_delta2 = base_df.copy() @@ -622,7 +627,7 @@ def test_11_jacobian_eq_delta_and_lin_to_quad(self): # -- ~GUI inputs: selection of functions func_df = pd.DataFrame(columns=['variable', 'ftype', 'weight', 'aggr']) - func_df['variable'] = ['ineq_cst', 'ineq_cst_array', 'eqcst_delta', 'eqcst_delta2', + func_df['variable'] = ['ineq_cst', 'ineq_cst0', 'eqcst_delta', 'eqcst_delta2', 'eqcst_delta_array', 'eqcst_lintoquad', 'eqcst_lintoquad_array', 'obj1', 'obj2'] func_df['ftype'] = [INEQ_CONSTRAINT, INEQ_CONSTRAINT, @@ -637,7 +642,7 @@ def test_11_jacobian_eq_delta_and_lin_to_quad(self): # -- data to simulate disciplinary chain outputs values_dict[prefix + 'ineq_cst'] = ineq_cst - values_dict[prefix + 'ineq_cst_array'] = ineq_cst_array + values_dict[prefix + 'ineq_cst0'] = ineq_cst0 values_dict[prefix + 'eqcst_delta'] = eqcst_delta values_dict[prefix + 'eqcst_delta2'] = eqcst_delta2 values_dict[prefix + 'eqcst_delta_array'] = eqcst_delta_array @@ -650,7 +655,7 @@ def test_11_jacobian_eq_delta_and_lin_to_quad(self): ee.load_study_from_input_dict(values_dict) - ee.dm.set_data(prefix + 'ineq_cst_array', 'type', 'array') + ee.dm.set_data(prefix + 'ineq_cst0', 'type', 'array') ee.dm.set_data(prefix + 'eqcst_delta_array', 'type', 'array') ee.dm.set_data(prefix + 'eqcst_lintoquad_array', 'type', 'array') @@ -678,7 +683,7 @@ def test_11_jacobian_eq_delta_and_lin_to_quad(self): assert disc_techno.check_jacobian( input_data=disc_techno.local_data, threshold=1e-8, inputs=['FuncManagerTest.FunctionManager.ineq_cst', - 'FuncManagerTest.FunctionManager.ineq_cst_array', + 'FuncManagerTest.FunctionManager.ineq_cst0', 'FuncManagerTest.FunctionManager.eqcst_delta', 'FuncManagerTest.FunctionManager.eqcst_delta2', 'FuncManagerTest.FunctionManager.eqcst_delta_array', @@ -726,3 +731,224 @@ def test_12_test_number_iteration_output_optim_df(self): func_disc = self.ee.dm.get_disciplines_with_name(f'{self.name}.{optim_name}.SellarCoupling.FunctionManager')[0] filter = func_disc.get_chart_filter_list() graph_list = func_disc.get_post_processing_list(filter) + + def test_13_jacobian_func_manager_disc_ineq_constraint_negative_when_satisfied(self): + INEQ_CONSTRAINT = self.func_manager.INEQ_CONSTRAINT + + # -- init the case + func_mng_name = 'FunctionManager' + prefix = self.name + '.' + func_mng_name + '.' + + ee = ExecutionEngine(self.name) + ns_dict = {'ns_functions': self.name + '.' + func_mng_name, + 'ns_optim': self.name + '.' + func_mng_name} + ee.ns_manager.add_ns_def(ns_dict) + + mod_list = 'sostrades_optimization_plugins.models.func_manager.func_manager_disc.FunctionManagerDisc' + fm_builder = ee.factory.get_builder_from_module( + 'FunctionManager', mod_list) + ee.factory.set_builders_to_coupling_builder(fm_builder) + ee.configure() + + # -- i/o setup + base_df = pd.DataFrame({'years': arange(10, 13)}) + cst0 = base_df.copy() + cst0['cst0_values'] = np.array([-10., 1, -5.]) + + # -- ~GUI inputs: selection of functions + + func_df = pd.DataFrame(columns=['variable', 'ftype', 'weight']) + func_df['variable'] = ['cst0'] + func_df['ftype'] = [INEQ_CONSTRAINT] + func_df['weight'] = [2.] + func_df['aggr'] = [FunctionManager.INEQ_NEGATIVE_WHEN_SATIFIED] + func_df['parent'] = 'obj' + func_df['namespace'] = '' + + + values_dict = {} + values_dict[prefix + FunctionManagerDisc.FUNC_DF] = func_df + + # -- data to simulate disciplinary chain outputs + values_dict[prefix + 'cst0'] = cst0 + + ee.load_study_from_input_dict(values_dict) + + ee.display_treeview_nodes(True) + + # -- execution + ee.execute() + # -- retrieve outputs + disc = ee.dm.get_disciplines_with_name( + f'{self.name}.{func_mng_name}')[0] + disc_techno = ee.root_process.proxy_disciplines[0].mdo_discipline_wrapp.mdo_discipline + + assert disc_techno.check_jacobian( + input_data=disc_techno.local_data, + threshold=1e-5, inputs=['FuncManagerTest.FunctionManager.cst0'], + outputs=['FuncManagerTest.FunctionManager.objective_lagrangian'], derr_approx='complex_step', step=1e-15) + + def test_14_jacobian_func_manager_disc_ineq_constraint_positive_when_satisfied(self): + INEQ_CONSTRAINT = self.func_manager.INEQ_CONSTRAINT + + # -- init the case + func_mng_name = 'FunctionManager' + prefix = self.name + '.' + func_mng_name + '.' + + ee = ExecutionEngine(self.name) + ns_dict = {'ns_functions': self.name + '.' + func_mng_name, + 'ns_optim': self.name + '.' + func_mng_name} + ee.ns_manager.add_ns_def(ns_dict) + + mod_list = 'sostrades_optimization_plugins.models.func_manager.func_manager_disc.FunctionManagerDisc' + fm_builder = ee.factory.get_builder_from_module( + 'FunctionManager', mod_list) + ee.factory.set_builders_to_coupling_builder(fm_builder) + ee.configure() + + # -- i/o setup + base_df = pd.DataFrame({'years': arange(10, 13)}) + cst0 = base_df.copy() + cst0['cst0_values'] = np.array([-10., 0.2, -5.]) + + # -- ~GUI inputs: selection of functions + + func_df = pd.DataFrame(columns=['variable', 'ftype', 'weight']) + func_df['variable'] = ['cst0'] + func_df['ftype'] = [INEQ_CONSTRAINT] + func_df['weight'] = [3.] + func_df['aggr'] = [FunctionManager.INEQ_POSITIVE_WHEN_SATIFIED] + func_df['parent'] = 'obj' + func_df['namespace'] = '' + + values_dict = {} + values_dict[prefix + FunctionManagerDisc.FUNC_DF] = func_df + + # -- data to simulate disciplinary chain outputs + values_dict[prefix + 'cst0'] = cst0 + + ee.load_study_from_input_dict(values_dict) + + ee.display_treeview_nodes(True) + + # -- execution + ee.execute() + # -- retrieve outputs + disc = ee.dm.get_disciplines_with_name( + f'{self.name}.{func_mng_name}')[0] + disc_techno = ee.root_process.proxy_disciplines[0].mdo_discipline_wrapp.mdo_discipline + + assert disc_techno.check_jacobian( + input_data=disc_techno.local_data, + threshold=1e-5, inputs=['FuncManagerTest.FunctionManager.cst0'], + outputs=['FuncManagerTest.FunctionManager.objective_lagrangian'], derr_approx='complex_step') + + def test_16_jacobian_func_manager_disc_ineq_constraint_negative_when_satisfied_square(self): + INEQ_CONSTRAINT = self.func_manager.INEQ_CONSTRAINT + + # -- init the case + func_mng_name = 'FunctionManager' + prefix = self.name + '.' + func_mng_name + '.' + + ee = ExecutionEngine(self.name) + ns_dict = {'ns_functions': self.name + '.' + func_mng_name, + 'ns_optim': self.name + '.' + func_mng_name} + ee.ns_manager.add_ns_def(ns_dict) + + mod_list = 'sostrades_optimization_plugins.models.func_manager.func_manager_disc.FunctionManagerDisc' + fm_builder = ee.factory.get_builder_from_module( + 'FunctionManager', mod_list) + ee.factory.set_builders_to_coupling_builder(fm_builder) + ee.configure() + + # -- i/o setup + base_df = pd.DataFrame({'years': arange(10, 13)}) + cst0 = base_df.copy() + cst0['cst0_values'] = np.array([-10., 0.2, -5.]) + + # -- ~GUI inputs: selection of functions + + func_df = pd.DataFrame(columns=['variable', 'ftype', 'weight']) + func_df['variable'] = ['cst0'] + func_df['ftype'] = [INEQ_CONSTRAINT] + func_df['weight'] = [2.5] + func_df['aggr'] = [FunctionManager.INEQ_NEGATIVE_WHEN_SATIFIED_AND_SQUARE_IT] + func_df['parent'] = 'obj' + func_df['namespace'] = '' + + values_dict = {} + values_dict[prefix + FunctionManagerDisc.FUNC_DF] = func_df + + # -- data to simulate disciplinary chain outputs + values_dict[prefix + 'cst0'] = cst0 + + ee.load_study_from_input_dict(values_dict) + + ee.display_treeview_nodes(True) + + # -- execution + ee.execute() + # -- retrieve outputs + disc = ee.dm.get_disciplines_with_name( + f'{self.name}.{func_mng_name}')[0] + disc_techno = ee.root_process.proxy_disciplines[0].mdo_discipline_wrapp.mdo_discipline + + assert disc_techno.check_jacobian( + input_data=disc_techno.local_data, + threshold=1e-5, inputs=['FuncManagerTest.FunctionManager.cst0'], + outputs=['FuncManagerTest.FunctionManager.objective_lagrangian'], derr_approx='complex_step') + + def test_17_jacobian_func_manager_disc_ineq_constraint_positive_when_satisfied_square(self): + INEQ_CONSTRAINT = self.func_manager.INEQ_CONSTRAINT + + # -- init the case + func_mng_name = 'FunctionManager' + prefix = self.name + '.' + func_mng_name + '.' + + ee = ExecutionEngine(self.name) + ns_dict = {'ns_functions': self.name + '.' + func_mng_name, + 'ns_optim': self.name + '.' + func_mng_name} + ee.ns_manager.add_ns_def(ns_dict) + + mod_list = 'sostrades_optimization_plugins.models.func_manager.func_manager_disc.FunctionManagerDisc' + fm_builder = ee.factory.get_builder_from_module( + 'FunctionManager', mod_list) + ee.factory.set_builders_to_coupling_builder(fm_builder) + ee.configure() + + # -- i/o setup + base_df = pd.DataFrame({'years': arange(10, 13)}) + cst0 = base_df.copy() + cst0['cst0_values'] = np.array([-10., 0.2, -5.]) + + # -- ~GUI inputs: selection of functions + + func_df = pd.DataFrame(columns=['variable', 'ftype', 'weight']) + func_df['variable'] = ['cst0'] + func_df['ftype'] = [INEQ_CONSTRAINT] + func_df['weight'] = [2.] + func_df['aggr'] = [FunctionManager.INEQ_POSITIVE_WHEN_SATIFIED_AND_SQUARE_IT] + func_df['parent'] = 'obj' + func_df['namespace'] = '' + + values_dict = {} + values_dict[prefix + FunctionManagerDisc.FUNC_DF] = func_df + + # -- data to simulate disciplinary chain outputs + values_dict[prefix + 'cst0'] = cst0 + + ee.load_study_from_input_dict(values_dict) + + ee.display_treeview_nodes(True) + + # -- execution + ee.execute() + # -- retrieve outputs + disc = ee.dm.get_disciplines_with_name( + f'{self.name}.{func_mng_name}')[0] + disc_techno = ee.root_process.proxy_disciplines[0].mdo_discipline_wrapp.mdo_discipline + + assert disc_techno.check_jacobian( + input_data=disc_techno.local_data, + threshold=1e-5, inputs=['FuncManagerTest.FunctionManager.cst0'], + outputs=['FuncManagerTest.FunctionManager.objective_lagrangian'], derr_approx='complex_step') diff --git a/sostrades_optimization_plugins/tests/l0_test_62_design_var_in_sellar.py b/sostrades_optimization_plugins/tests/l0_test_62_design_var_in_sellar.py index 275aca6..e6adcee 100644 --- a/sostrades_optimization_plugins/tests/l0_test_62_design_var_in_sellar.py +++ b/sostrades_optimization_plugins/tests/l0_test_62_design_var_in_sellar.py @@ -43,8 +43,8 @@ class TestDesignVar(AbstractJacobianUnittest): """ def analytic_grad_entry(self): - return [self.test_derivative - ] + return [] + def setUp(self): @@ -170,9 +170,3 @@ def test_derivative(self): discipline=disc.mdo_discipline_wrapp.mdo_discipline, step=1e-15, inputs=input_names, outputs=output_names, derr_approx='complex_step') - -if '__main__' == __name__: - cls = TestDesignVar() - cls.setUp() - cls.test_01_check_execute() - cls.test_derivative() diff --git a/sostrades_optimization_plugins/tests/l0_test_63_design_var.py b/sostrades_optimization_plugins/tests/l0_test_63_design_var.py index 080fa16..5a0201d 100644 --- a/sostrades_optimization_plugins/tests/l0_test_63_design_var.py +++ b/sostrades_optimization_plugins/tests/l0_test_63_design_var.py @@ -29,9 +29,9 @@ class TestDesignVar(AbstractJacobianUnittest): """ DesignVar unitary test class """ + def analytic_grad_entry(self): - return [self.test_derivative - ] + return [] def setUp(self): self.study_name = 'Test' @@ -206,9 +206,3 @@ def test_03_check_deactivated_element_options(self): z_in_value = list(self.values_dict[f'{self.ns}.z_in']) z_value_th = z_in_value + [z_in_value[-1]] * (len(dspace_z_value) - len(z_in_value)) assert (list(z) == z_value_th) - - -if '__main__' == __name__: - cls = TestDesignVar() - cls.setUp() - cls.test_01_check_execute() diff --git a/sostrades_optimization_plugins/tests/l1_test_01_gradient_design_var_disc.py b/sostrades_optimization_plugins/tests/l1_test_01_gradient_design_var_disc.py index 3cd0682..b2fa612 100644 --- a/sostrades_optimization_plugins/tests/l1_test_01_gradient_design_var_disc.py +++ b/sostrades_optimization_plugins/tests/l1_test_01_gradient_design_var_disc.py @@ -33,11 +33,9 @@ class GradiantAssetDiscTestCase(AbstractJacobianUnittest): """ np.random.seed(42) + def analytic_grad_entry(self): - return [ - self.test_01_analytic_gradient_default_dataframe_fill(), - self.test_02_analytic_gradient_dataframe_fill_one_column_for_key(), - ] + return [] def setUp(self): """Initialize""" diff --git a/sostrades_optimization_plugins/tests/l1_test_gradient_sellar_optim_subprocess.py b/sostrades_optimization_plugins/tests/l1_test_gradient_sellar_optim_subprocess.py index ceb017d..674a48b 100644 --- a/sostrades_optimization_plugins/tests/l1_test_gradient_sellar_optim_subprocess.py +++ b/sostrades_optimization_plugins/tests/l1_test_gradient_sellar_optim_subprocess.py @@ -55,9 +55,9 @@ class SellarOptimSubprocessJacobianDiscTest(AbstractJacobianUnittest): - test_11_gradient_subprocess_wo_dvar_fmanager_flatten_local_data : Test Sellar without design var nor func manager """ + def analytic_grad_entry(self): - return [self._test_01_gradient_subprocess_double_level_coupling(), - ] + return [] def setUp(self): self.name = 'Test' diff --git a/sostrades_optimization_plugins/tools/cst_manager/func_manager_common.py b/sostrades_optimization_plugins/tools/cst_manager/func_manager_common.py index ba0e1fb..5a18f3f 100644 --- a/sostrades_optimization_plugins/tools/cst_manager/func_manager_common.py +++ b/sostrades_optimization_plugins/tools/cst_manager/func_manager_common.py @@ -16,6 +16,8 @@ ''' # pylint: disable=unsubscriptable-object +import warnings + import numpy as np from sostrades_core.tools.base_functions.exp_min import ( compute_dfunc_with_exp_min, @@ -218,7 +220,7 @@ def get_dcons_smooth_dvariable_vect(cst, alpha=1E16): alphaxcst = alpha * cst_array max_alphax = np.amax(alphaxcst, axis=1) - + warnings.filterwarnings("ignore", category=RuntimeWarning) k = max_alphax - max_exp exp_func = np.maximum(min_exp, alpha * cst_array - np.repeat(k, cst_array.shape[1]).reshape(cst_array.shape)) @@ -256,4 +258,59 @@ def get_dcons_smooth_dvariable_vect(cst, alpha=1E16): for i in np.arange(cst_array.shape[0]): grad_value[i][np.logical_not(non_max_idx)[i]] = grad_val_max[i] + warnings.filterwarnings("ignore", category=RuntimeWarning) return grad_value + + +def pseudo_abs_obj(value: np.ndarray, eps= 1e-2): + """compute the pseudo absolute value of x, where x is a 1dimensional array""" + return np.array([np.sum(np.sqrt(value ** 2 + eps ** 2) - eps)]) / len(value) + + +def d_pseudo_abs_obj(value, d_value, eps=1e-2): + """compute the derivative of the pseudo absolute value of x, where x is a 1dimensional array""" + return d_value.T @ (value / np.sqrt(value ** 2 + eps ** 2)) / len(value) + + +def keep_positive_only(value: np.ndarray): + """values must be a 1dimensional np array""" + result = np.maximum(value, 0) + return result.mean() + + +def keep_negative_only(value: np.ndarray): + """values must be a 1dimensional np array""" + result = np.minimum(value, 0) + return result.mean() + + +def derivative_d_keep_positive_only(value: np.ndarray): + """values must be a 1dimensional np array""" + return ((value > 0) * 1) / len(value) + + +def derivative_d_keep_negative_only(value: np.ndarray): + """values must be a 1dimensional np array""" + return ((value < 0) * 1) / len(value) + + +def keep_positive_only_square(value: np.ndarray): + """values must be a 1dimensional np array""" + result = np.maximum(value, 0) ** 2 + return result.mean() + + +def keep_negative_only_square(value: np.ndarray): + """values must be a 1dimensional np array""" + result = np.minimum(value, 0) ** 2 + return result.mean() + + +def derivative_d_keep_positive_only_square(value: np.ndarray): + """values must be a 1dimensional np array""" + return ((value > 0) * 1) * value * 2/ len(value) + + +def derivative_d_keep_negative_only_square(value: np.ndarray): + """values must be a 1dimensional np array""" + return ((value < 0) * 1) * value * 2/ len(value)