fix bug for utf-8

alibaba · Dec 19, 2023 · e9fc1cc · e9fc1cc
1 parent c31ae3c
commit e9fc1cc
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py
@@ -91,6 +91,10 @@ def _odps_read(self):
                   second_split)
             if one_data[col_id] not in ['', 'NULL', None]:
               batch_data_np[col_id][row_id] = one_data[col_id]
+              print('here0, batch_data_np = ', batch_data_np)
+              one_data[col_id] = one_data[col_id].decode('utf-8')
+              batch_data_np[col_id][row_id] = one_data[col_id]
+              print('here1, batch_data_np = ', batch_data_np)
             print('batch_data_np = ', batch_data_np)
         yield tuple(batch_data_np)
       if res_num > 0:
@@ -109,6 +113,10 @@ def _odps_read(self):
                   second_split)
             if one_data[col_id] not in ['', 'NULL', None]:
               batch_data_np[col_id][row_id] = one_data[col_id]
+              print('here00, batch_data_np = ', batch_data_np)
+              one_data[col_id] = one_data[col_id].decode('utf-8')
+              batch_data_np[col_id][row_id] = one_data[col_id]
+              print('here11, batch_data_np = ', batch_data_np)
             print('batch_data_np = ', batch_data_np)
         yield tuple(batch_data_np)
       reader.close()