dealing with missing data

MRCIEU · May 21, 2021 · e89961b · e89961b
1 parent 5706e49
commit e89961b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/create/index_semmeddb_citations.py b/create/index_semmeddb_citations.py
@@ -77,6 +77,7 @@ def index_sentence_data(sentence_data, index_name):
         "PYEAR"
     ]
     df.columns = col_names
+    df.fillna('NA',inplace=True)
     logger.info(f"\n{df.head()}")
     logger.info(df.shape)
 
@@ -99,15 +100,16 @@ def index_sentence_data(sentence_data, index_name):
             )
             bulk_data = []
         # print(line.decode('utf-8'))
-        PMID = row['PMID'].replace("'", "")
-        if PMID in pmids:
+        #PMID = row['PMID'].replace("'", "")
+        if str(row['PMID']) in pmids:
             data_dict = {
-                "PMID": PMID,
+                "PMID": row['PMID'],
                 "ISSN": row['ISSN'],
                 "DP": row['DP'],
                 "EDAT": row['EDAT'],
                 "PYEAR": int(row['PYEAR']),
             }
+
             op_dict = {
                 "_index": index_name,
                 # "_id": l[0],

diff --git a/create/index_semmeddb_sentences.py b/create/index_semmeddb_sentences.py
@@ -105,7 +105,8 @@ def index_sentence_data(sentence_data, index_name):
     ]
     df.columns = col_names
     df.drop(columns=["SECTION_HEADER", "NORMALIZED_SECTION_HEADER"], inplace=True)
-    df.dropna(inplace=True)
+    #df.dropna(inplace=True)
+    df.fillna('NA',inplace=True)
     logger.info(f"\n{df.head()}")
     logger.info(df.shape)
     for i, row in df.iterrows():