From cf4c8a210c34d2b8de7fb5800c99882c3a16d62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=AB=98=E9=A3=9E?= <1052924341@qq.com> Date: Fri, 5 Jul 2024 15:31:25 +0800 Subject: [PATCH 1/2] fix update_config_remove_invalid_subseq_feature --- easy_rec/python/compat/early_stopping.py | 2 +- easy_rec/python/test/train_eval_test.py | 2 +- .../tools/add_feature_info_to_config.py | 24 +++++++++++++++++++ setup.cfg | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fc850fb62..fe4c12132 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,9 +21,9 @@ import os import threading import time +from distutils.version import LooseVersion import tensorflow as tf -from distutils.version import LooseVersion from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import init_ops diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 68d0b8656..bf2052dc5 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,11 +7,11 @@ import threading import time import unittest +from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf -from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index f1b4a4cfd..9848d5dfd 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -111,6 +111,30 @@ def main(argv): logging.info('drop feature: %s' % feature_name) feature_group.ClearField('feature_names') feature_group.feature_names.extend(reserved_features) + for sequence_feature in feature_group.sequence_features: + seq_att_maps = sequence_feature.seq_att_map + for seq_att in seq_att_maps: + keys = seq_att.key + reserved_keys = [] + for key in keys: + if key not in drop_feature_names: + reserved_keys.append(key) + else: + logging.info('drop sequence feature key: %s' % key) + seq_att.ClearField('key') + seq_att.key.extend(reserved_keys) + + hist_seqs = seq_att.hist_seq + reserved_hist_seqs = [] + for hist_seq in hist_seqs: + if 'seq__' in hist_seq: + hist_seq = hist_seq.split('seq__')[1] + if hist_seq not in drop_feature_names: + reserved_hist_seqs.append(hist_seq) + else: + logging.info('drop sequence feature hist_seq: %s' % hist_seq) + seq_att.ClearField('hist_seq') + seq_att.hist_seq.extend(reserved_hist_seqs) config_dir, config_name = os.path.split(FLAGS.output_config_path) config_util.save_pipeline_config(pipeline_config, config_dir, config_name) diff --git a/setup.cfg b/setup.cfg index b43211827..337833a0f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ multi_line_output = 7 force_single_line = true known_standard_library = setuptools known_first_party = easy_rec -known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml +known_third_party = absl,common_io,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml no_lines_before = LOCALFOLDER default_section = THIRDPARTY skip = easy_rec/python/protos From dcb0cf4f65c7cd20138449501da966cb34f18aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=AB=98=E9=A3=9E?= <1052924341@qq.com> Date: Mon, 8 Jul 2024 10:34:07 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=A0=E9=99=A4model?= =?UTF-8?q?=5Fconfig=E4=B8=AD=E5=BE=97hist=5Fseq?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- easy_rec/python/compat/early_stopping.py | 2 +- easy_rec/python/test/train_eval_test.py | 2 +- easy_rec/python/tools/add_feature_info_to_config.py | 2 -- setup.cfg | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fe4c12132..fc850fb62 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,9 +21,9 @@ import os import threading import time -from distutils.version import LooseVersion import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import init_ops diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index bf2052dc5..68d0b8656 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,11 +7,11 @@ import threading import time import unittest -from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index 9848d5dfd..b11cfc0a7 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -127,8 +127,6 @@ def main(argv): hist_seqs = seq_att.hist_seq reserved_hist_seqs = [] for hist_seq in hist_seqs: - if 'seq__' in hist_seq: - hist_seq = hist_seq.split('seq__')[1] if hist_seq not in drop_feature_names: reserved_hist_seqs.append(hist_seq) else: diff --git a/setup.cfg b/setup.cfg index 337833a0f..b43211827 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ multi_line_output = 7 force_single_line = true known_standard_library = setuptools known_first_party = easy_rec -known_third_party = absl,common_io,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml +known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml no_lines_before = LOCALFOLDER default_section = THIRDPARTY skip = easy_rec/python/protos