From 0108c2289656789dd18558313a5e0a8286bda001 Mon Sep 17 00:00:00 2001 From: wwxxzz Date: Fri, 20 Oct 2023 16:52:41 +0800 Subject: [PATCH 1/3] remove feature_configs for drop features --- easy_rec/python/tools/add_feature_info_to_config.py | 6 +++--- easy_rec/python/utils/config_util.py | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index e82a04516..1c2d9b606 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -59,9 +59,9 @@ def main(argv): except common_io.exception.OutOfRangeException: reader.close() break - - for feature_config in config_util.get_compatible_feature_configs( - pipeline_config): + + feature_configs = config_util.get_compatible_feature_configs(pipeline_config, drop_feature_names) + for feature_config in feature_configs: feature_name = feature_config.input_names[0] if feature_name in feature_info_map: logging.info('edited %s' % feature_name) diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index b63a02f71..8443226c7 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -390,14 +390,19 @@ def add_boundaries_to_config(pipeline_config, tables): logging.info('edited %s' % feature_name) -def get_compatible_feature_configs(pipeline_config): +def get_compatible_feature_configs(pipeline_config,drop_feature_names=None): if pipeline_config.feature_configs: feature_configs = pipeline_config.feature_configs else: feature_configs = pipeline_config.feature_config.features + if drop_feature_names: + tmp_feature_configs = feature_configs[:] + for fea_cfg in tmp_feature_configs: + fea_name = fea_cfg.input_names[0] + if fea_name in drop_feature_names: + feature_configs.remove(fea_cfg) return feature_configs - def parse_time(time_data): """Parse time string to timestamp. From 17f67b1efdaa8de962d36664ac05a83726a1a63f Mon Sep 17 00:00:00 2001 From: wwxxzz Date: Mon, 23 Oct 2023 15:47:22 +0800 Subject: [PATCH 2/3] put the code 'remove feature_configs for drop features' in add_fea_info_to_cfg.py --- easy_rec/python/tools/add_feature_info_to_config.py | 8 +++++++- easy_rec/python/utils/config_util.py | 8 +------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index 1c2d9b606..a2df7744a 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -60,7 +60,13 @@ def main(argv): reader.close() break - feature_configs = config_util.get_compatible_feature_configs(pipeline_config, drop_feature_names) + feature_configs = config_util.get_compatible_feature_configs(pipeline_config) + if drop_feature_names: + tmp_feature_configs = feature_configs[:] + for fea_cfg in tmp_feature_configs: + fea_name = fea_cfg.input_names[0] + if fea_name in drop_feature_names: + feature_configs.remove(fea_cfg) for feature_config in feature_configs: feature_name = feature_config.input_names[0] if feature_name in feature_info_map: diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 8443226c7..5e9dea852 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -390,17 +390,11 @@ def add_boundaries_to_config(pipeline_config, tables): logging.info('edited %s' % feature_name) -def get_compatible_feature_configs(pipeline_config,drop_feature_names=None): +def get_compatible_feature_configs(pipeline_config): if pipeline_config.feature_configs: feature_configs = pipeline_config.feature_configs else: feature_configs = pipeline_config.feature_config.features - if drop_feature_names: - tmp_feature_configs = feature_configs[:] - for fea_cfg in tmp_feature_configs: - fea_name = fea_cfg.input_names[0] - if fea_name in drop_feature_names: - feature_configs.remove(fea_cfg) return feature_configs def parse_time(time_data): From bf47aac67a00e7c44cdda5a33a70db9d522b8213 Mon Sep 17 00:00:00 2001 From: wwxxzz Date: Mon, 23 Oct 2023 15:48:50 +0800 Subject: [PATCH 3/3] put the code 'remove feature_configs for drop features' in add_fea_info_to_cfg.py --- easy_rec/python/utils/config_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 5e9dea852..b63a02f71 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -397,6 +397,7 @@ def get_compatible_feature_configs(pipeline_config): feature_configs = pipeline_config.feature_config.features return feature_configs + def parse_time(time_data): """Parse time string to timestamp.