ml-for-missing-values.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":65862,"databundleVersionId":7469115,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#This Python 3 environment comes with many helpful analytics libraries installed\n#It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n#For example, here's several helpful packages to load\nimport numpy as np # linear algebra \nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n#Input data files are available in the read-only \"../input/\" directory\n#For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n\n#import os \n#for dirname, _, filenames in os.walk('/kaggle/input'): \n#    for filename in filenames: \n#        print(os.path.join(dirname, filename))\n\n#You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n#You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/widsdatathon2024-challenge1/training.csv')\ntest = pd.read_csv('/kaggle/input/widsdatathon2024-challenge1/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:04.546104Z","iopub.execute_input":"2024-02-15T19:55:04.546777Z","iopub.status.idle":"2024-02-15T19:55:04.968868Z","shell.execute_reply.started":"2024-02-15T19:55:04.546739Z","shell.execute_reply":"2024-02-15T19:55:04.967653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's impute 29 missing data points for Ozone, PM25 and NO2.\n## I'll use XGBRegressor to predict the missing values","metadata":{}},{"cell_type":"code","source":"train.drop(columns = ['metastatic_first_novel_treatment', 'metastatic_first_novel_treatment_type', 'patient_race', 'payer_type' , 'bmi'], inplace= True)\ntest.drop(columns = ['metastatic_first_novel_treatment', 'metastatic_first_novel_treatment_type', 'patient_race', 'payer_type' , 'bmi'], inplace= True)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:04.970814Z","iopub.execute_input":"2024-02-15T19:55:04.971166Z","iopub.status.idle":"2024-02-15T19:55:04.988426Z","shell.execute_reply.started":"2024-02-15T19:55:04.971134Z","shell.execute_reply":"2024-02-15T19:55:04.986889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['DiagPeriodL90D']=train['DiagPeriodL90D'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:04.989671Z","iopub.execute_input":"2024-02-15T19:55:04.989997Z","iopub.status.idle":"2024-02-15T19:55:04.997233Z","shell.execute_reply.started":"2024-02-15T19:55:04.989968Z","shell.execute_reply":"2024-02-15T19:55:04.996412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 83)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.000444Z","iopub.execute_input":"2024-02-15T19:55:05.000809Z","iopub.status.idle":"2024-02-15T19:55:05.008808Z","shell.execute_reply.started":"2024-02-15T19:55:05.000776Z","shell.execute_reply":"2024-02-15T19:55:05.007453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.concat([ train.drop('DiagPeriodL90D', axis= 1), test], axis = 0)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.010395Z","iopub.execute_input":"2024-02-15T19:55:05.010711Z","iopub.status.idle":"2024-02-15T19:55:05.048643Z","shell.execute_reply.started":"2024-02-15T19:55:05.010683Z","shell.execute_reply":"2024-02-15T19:55:05.047730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ad_types = all_data.dtypes.to_dict()\nfor key in ad_types:\n    if ad_types[key]=='O':\n        all_data[key] = all_data[key].astype('category')\n","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.049888Z","iopub.execute_input":"2024-02-15T19:55:05.050901Z","iopub.status.idle":"2024-02-15T19:55:05.083858Z","shell.execute_reply.started":"2024-02-15T19:55:05.050866Z","shell.execute_reply":"2024-02-15T19:55:05.082792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#changing patient_zip3 into categorical variable\n\nall_data['patient_zip3']=all_data['patient_zip3'].astype('category')\nall_data.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.085248Z","iopub.execute_input":"2024-02-15T19:55:05.086138Z","iopub.status.idle":"2024-02-15T19:55:05.100303Z","shell.execute_reply.started":"2024-02-15T19:55:05.086105Z","shell.execute_reply":"2024-02-15T19:55:05.099271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, StratifiedKFold, RandomizedSearchCV\nfrom sklearn.metrics import confusion_matrix, roc_auc_score\nfrom xgboost import XGBClassifier\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.101924Z","iopub.execute_input":"2024-02-15T19:55:05.102262Z","iopub.status.idle":"2024-02-15T19:55:05.694415Z","shell.execute_reply.started":"2024-02-15T19:55:05.102226Z","shell.execute_reply":"2024-02-15T19:55:05.693140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss_ozone = all_data['Ozone'].isna()\nmiss_PM25 = all_data['PM25'].isna()\nmiss_NO2 = all_data['N02'].isna()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.699084Z","iopub.execute_input":"2024-02-15T19:55:05.699644Z","iopub.status.idle":"2024-02-15T19:55:05.709090Z","shell.execute_reply.started":"2024-02-15T19:55:05.699597Z","shell.execute_reply":"2024-02-15T19:55:05.707434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ozone_data = all_data[miss_ozone]\nPM25_data = all_data[miss_PM25]\nNO2_data = all_data[miss_NO2]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.711426Z","iopub.execute_input":"2024-02-15T19:55:05.711999Z","iopub.status.idle":"2024-02-15T19:55:05.725892Z","shell.execute_reply.started":"2024-02-15T19:55:05.711953Z","shell.execute_reply":"2024-02-15T19:55:05.724162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ozone_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.728041Z","iopub.execute_input":"2024-02-15T19:55:05.728595Z","iopub.status.idle":"2024-02-15T19:55:05.775236Z","shell.execute_reply.started":"2024-02-15T19:55:05.728507Z","shell.execute_reply":"2024-02-15T19:55:05.773908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"air_polution_traindata =  all_data[~miss_ozone]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.777152Z","iopub.execute_input":"2024-02-15T19:55:05.777960Z","iopub.status.idle":"2024-02-15T19:55:05.809983Z","shell.execute_reply.started":"2024-02-15T19:55:05.777916Z","shell.execute_reply":"2024-02-15T19:55:05.808604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nimport xgboost as xgb\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import RandomizedSearchCV,GridSearchCV","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.815983Z","iopub.execute_input":"2024-02-15T19:55:05.816371Z","iopub.status.idle":"2024-02-15T19:55:05.822938Z","shell.execute_reply.started":"2024-02-15T19:55:05.816340Z","shell.execute_reply":"2024-02-15T19:55:05.821546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = air_polution_traindata.drop(columns=['Ozone',  'patient_id'])\ny = air_polution_traindata['Ozone']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.824230Z","iopub.execute_input":"2024-02-15T19:55:05.824611Z","iopub.status.idle":"2024-02-15T19:55:05.848966Z","shell.execute_reply.started":"2024-02-15T19:55:05.824582Z","shell.execute_reply":"2024-02-15T19:55:05.847652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.4, shuffle=True, random_state=66)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.850454Z","iopub.execute_input":"2024-02-15T19:55:05.850791Z","iopub.status.idle":"2024-02-15T19:55:05.870018Z","shell.execute_reply.started":"2024-02-15T19:55:05.850761Z","shell.execute_reply":"2024-02-15T19:55:05.868620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm = xgb.XGBRegressor(enable_categorical = True)\nreg_cv = GridSearchCV(gbm, {\"colsample_bytree\":[1.0],\"min_child_weight\":[1.0,1.2]\n                            ,'max_depth': [3,4,6], 'n_estimators': [500,1000]}, verbose=3)\nreg_cv.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:55:05.872229Z","iopub.execute_input":"2024-02-15T19:55:05.872773Z","iopub.status.idle":"2024-02-15T19:58:48.105259Z","shell.execute_reply.started":"2024-02-15T19:55:05.872725Z","shell.execute_reply":"2024-02-15T19:58:48.103805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_cv.best_params_","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:48.106751Z","iopub.execute_input":"2024-02-15T19:58:48.107101Z","iopub.status.idle":"2024-02-15T19:58:48.116066Z","shell.execute_reply.started":"2024-02-15T19:58:48.107071Z","shell.execute_reply":"2024-02-15T19:58:48.114645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm_ozone_pred = xgb.XGBRegressor(**reg_cv.best_params_, enable_categorical =  True)\ngbm_ozone_pred.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:48.117723Z","iopub.execute_input":"2024-02-15T19:58:48.118045Z","iopub.status.idle":"2024-02-15T19:58:53.941153Z","shell.execute_reply.started":"2024-02-15T19:58:48.118018Z","shell.execute_reply":"2024-02-15T19:58:53.940227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = gbm_ozone_pred.predict(X_test)\ngbm_ozone_pred.score(X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:53.943049Z","iopub.execute_input":"2024-02-15T19:58:53.943845Z","iopub.status.idle":"2024-02-15T19:58:54.162845Z","shell.execute_reply.started":"2024-02-15T19:58:53.943811Z","shell.execute_reply":"2024-02-15T19:58:54.161403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ozone_pred = gbm_ozone_pred.predict(ozone_data.drop(columns=['Ozone','patient_id']))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:54.164333Z","iopub.execute_input":"2024-02-15T19:58:54.164662Z","iopub.status.idle":"2024-02-15T19:58:54.192361Z","shell.execute_reply.started":"2024-02-15T19:58:54.164632Z","shell.execute_reply":"2024-02-15T19:58:54.190802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(ozone_pred)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:54.194444Z","iopub.execute_input":"2024-02-15T19:58:54.195356Z","iopub.status.idle":"2024-02-15T19:58:54.206992Z","shell.execute_reply.started":"2024-02-15T19:58:54.195300Z","shell.execute_reply":"2024-02-15T19:58:54.201398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = air_polution_traindata.drop(columns=['N02',  'patient_id'])\ny = air_polution_traindata['N02']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:54.209251Z","iopub.execute_input":"2024-02-15T19:58:54.210161Z","iopub.status.idle":"2024-02-15T19:58:54.222448Z","shell.execute_reply.started":"2024-02-15T19:58:54.210121Z","shell.execute_reply":"2024-02-15T19:58:54.220692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.4, shuffle=True, random_state=66)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:54.223817Z","iopub.execute_input":"2024-02-15T19:58:54.224142Z","iopub.status.idle":"2024-02-15T19:58:54.241610Z","shell.execute_reply.started":"2024-02-15T19:58:54.224106Z","shell.execute_reply":"2024-02-15T19:58:54.240725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm = xgb.XGBRegressor(enable_categorical = True)\nreg_cv = GridSearchCV(gbm, {\"colsample_bytree\":[1.0],\"min_child_weight\":[1.0,1.2]\n                            ,'max_depth': [3,4,6], 'n_estimators': [500,1000]}, verbose=0)\nreg_cv.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T19:58:54.242921Z","iopub.execute_input":"2024-02-15T19:58:54.244395Z","iopub.status.idle":"2024-02-15T20:02:38.046687Z","shell.execute_reply.started":"2024-02-15T19:58:54.244361Z","shell.execute_reply":"2024-02-15T20:02:38.045664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_cv.best_params_","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:38.047801Z","iopub.execute_input":"2024-02-15T20:02:38.048578Z","iopub.status.idle":"2024-02-15T20:02:38.058813Z","shell.execute_reply.started":"2024-02-15T20:02:38.048521Z","shell.execute_reply":"2024-02-15T20:02:38.057491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm_no2_pred = xgb.XGBRegressor(**reg_cv.best_params_, enable_categorical =  True)\ngbm_no2_pred.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:38.060511Z","iopub.execute_input":"2024-02-15T20:02:38.060936Z","iopub.status.idle":"2024-02-15T20:02:42.777353Z","shell.execute_reply.started":"2024-02-15T20:02:38.060898Z","shell.execute_reply":"2024-02-15T20:02:42.776415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = gbm_no2_pred.predict(X_test)\ngbm_no2_pred.score(X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:42.778807Z","iopub.execute_input":"2024-02-15T20:02:42.779548Z","iopub.status.idle":"2024-02-15T20:02:42.948189Z","shell.execute_reply.started":"2024-02-15T20:02:42.779494Z","shell.execute_reply":"2024-02-15T20:02:42.947156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NO2_pred = gbm_no2_pred.predict(ozone_data.drop(columns=['N02','patient_id']))\nprint(NO2_pred)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:42.949776Z","iopub.execute_input":"2024-02-15T20:02:42.950537Z","iopub.status.idle":"2024-02-15T20:02:42.997472Z","shell.execute_reply.started":"2024-02-15T20:02:42.950496Z","shell.execute_reply":"2024-02-15T20:02:42.996419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = air_polution_traindata.drop(columns=['PM25',  'patient_id'])\ny = air_polution_traindata['PM25']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:42.999199Z","iopub.execute_input":"2024-02-15T20:02:42.999924Z","iopub.status.idle":"2024-02-15T20:02:43.013907Z","shell.execute_reply.started":"2024-02-15T20:02:42.999877Z","shell.execute_reply":"2024-02-15T20:02:43.012752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.4, shuffle=True, random_state=66)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:43.015615Z","iopub.execute_input":"2024-02-15T20:02:43.016011Z","iopub.status.idle":"2024-02-15T20:02:43.038134Z","shell.execute_reply.started":"2024-02-15T20:02:43.015974Z","shell.execute_reply":"2024-02-15T20:02:43.036792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm = xgb.XGBRegressor(enable_categorical = True)\nreg_cv = GridSearchCV(gbm, {\"colsample_bytree\":[1.0],\"min_child_weight\":[1.0,1.2]\n                            ,'max_depth': [3,4,6], 'n_estimators': [500,1000]}, verbose=0)\nreg_cv.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:02:43.039680Z","iopub.execute_input":"2024-02-15T20:02:43.040040Z","iopub.status.idle":"2024-02-15T20:06:12.810380Z","shell.execute_reply.started":"2024-02-15T20:02:43.040008Z","shell.execute_reply":"2024-02-15T20:06:12.809146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_cv.best_params_","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:12.812098Z","iopub.execute_input":"2024-02-15T20:06:12.812455Z","iopub.status.idle":"2024-02-15T20:06:12.819794Z","shell.execute_reply.started":"2024-02-15T20:06:12.812423Z","shell.execute_reply":"2024-02-15T20:06:12.818772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm_pm25_pred = xgb.XGBRegressor(**reg_cv.best_params_, enable_categorical =  True)\ngbm_pm25_pred.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:12.821799Z","iopub.execute_input":"2024-02-15T20:06:12.822929Z","iopub.status.idle":"2024-02-15T20:06:16.575791Z","shell.execute_reply.started":"2024-02-15T20:06:12.822894Z","shell.execute_reply":"2024-02-15T20:06:16.574596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = gbm_pm25_pred.predict(X_test)\ngbm_pm25_pred.score(X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:16.577971Z","iopub.execute_input":"2024-02-15T20:06:16.579565Z","iopub.status.idle":"2024-02-15T20:06:16.749647Z","shell.execute_reply.started":"2024-02-15T20:06:16.579520Z","shell.execute_reply":"2024-02-15T20:06:16.748093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PM25_pred = gbm_pm25_pred.predict(ozone_data.drop(columns=['PM25','patient_id']))\nprint(PM25_pred)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:16.751247Z","iopub.execute_input":"2024-02-15T20:06:16.752421Z","iopub.status.idle":"2024-02-15T20:06:16.809553Z","shell.execute_reply.started":"2024-02-15T20:06:16.752380Z","shell.execute_reply":"2024-02-15T20:06:16.808399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_air_index = all_data[all_data['Ozone'].isna()].index.tolist()\nprint(missing_air_index)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:16.810946Z","iopub.execute_input":"2024-02-15T20:06:16.811658Z","iopub.status.idle":"2024-02-15T20:06:16.822146Z","shell.execute_reply.started":"2024-02-15T20:06:16.811621Z","shell.execute_reply":"2024-02-15T20:06:16.820688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for count, idx in enumerate(missing_air_index):\n    all_data.loc[idx, 'Ozone'] = ozone_pred[count]\n    all_data.loc[idx, 'N02'] = NO2_pred[count]\n    all_data.loc[idx, 'PM25'] = PM25_pred[count]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:16.823962Z","iopub.execute_input":"2024-02-15T20:06:16.824407Z","iopub.status.idle":"2024-02-15T20:06:16.861808Z","shell.execute_reply.started":"2024-02-15T20:06:16.824367Z","shell.execute_reply":"2024-02-15T20:06:16.860808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.to_csv('all_data_airpol.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:16.863712Z","iopub.execute_input":"2024-02-15T20:06:16.864689Z","iopub.status.idle":"2024-02-15T20:06:19.182671Z","shell.execute_reply.started":"2024-02-15T20:06:16.864636Z","shell.execute_reply":"2024-02-15T20:06:19.181221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I'll impute missing geo data [patient_state , Region, Division] using patient_zip3 column","metadata":{}},{"cell_type":"code","source":"all_data.reset_index(inplace= True)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:19.184356Z","iopub.execute_input":"2024-02-15T20:06:19.184822Z","iopub.status.idle":"2024-02-15T20:06:19.194535Z","shell.execute_reply.started":"2024-02-15T20:06:19.184769Z","shell.execute_reply":"2024-02-15T20:06:19.192741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_state = all_data[all_data['patient_state'].isna()]['patient_zip3']\nmissing_region = all_data[all_data['Region'].isna()]['patient_zip3']\nmissing_division = all_data[all_data['Division'].isna()]['patient_zip3']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:19.212664Z","iopub.execute_input":"2024-02-15T20:06:19.213639Z","iopub.status.idle":"2024-02-15T20:06:19.226509Z","shell.execute_reply.started":"2024-02-15T20:06:19.213596Z","shell.execute_reply":"2024-02-15T20:06:19.225023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_region==missing_division","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:19.232992Z","iopub.execute_input":"2024-02-15T20:06:19.233409Z","iopub.status.idle":"2024-02-15T20:06:19.247355Z","shell.execute_reply.started":"2024-02-15T20:06:19.233374Z","shell.execute_reply":"2024-02-15T20:06:19.245633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_zip = {}\nfor ind in all_data.index:\n    if all_data['patient_zip3'][ind] not in dict_zip.keys():\n        dict_zip[all_data['patient_zip3'][ind]] = [all_data['patient_state'][ind], all_data['Division'][ind], all_data['Region'][ind]]\n    \nprint(dict_zip)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:19.249556Z","iopub.execute_input":"2024-02-15T20:06:19.250091Z","iopub.status.idle":"2024-02-15T20:06:19.691209Z","shell.execute_reply.started":"2024-02-15T20:06:19.250039Z","shell.execute_reply":"2024-02-15T20:06:19.689728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_zip[645] = ['MO', 'West North Central', 'Midwest']\ndict_zip[157] = ['PA', 'Middle Atlantic', 'Northeast']\ndict_zip[136] = ['NY', 'Middle Atlantic', 'Northeast']\ndict_zip[181] = ['PA', 'Middle Atlantic', 'Northeast']\ndict_zip[330] = ['FL', 'South Atlantic', 'South']\ndict_zip[973] = ['OR', 'Pacific', 'West']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:19.693054Z","iopub.execute_input":"2024-02-15T20:06:19.693564Z","iopub.status.idle":"2024-02-15T20:06:19.702644Z","shell.execute_reply.started":"2024-02-15T20:06:19.693520Z","shell.execute_reply":"2024-02-15T20:06:19.701175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ind in all_data.index:\n    if all_data['patient_state'][ind] is np.nan:\n        all_data['patient_state'][ind] = dict_zip[all_data['patient_zip3'][ind]][0]\n        \n    if all_data['Division'][ind] is np.nan:\n        all_data['Division'][ind] = dict_zip[all_data['patient_zip3'][ind]][1]\n    if all_data['Region'][ind] is np.nan:\n        all_data['Region'][ind] = dict_zip[all_data['patient_zip3'][ind]][2]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:19.704290Z","iopub.execute_input":"2024-02-15T20:06:19.704761Z","iopub.status.idle":"2024-02-15T20:06:20.782708Z","shell.execute_reply.started":"2024-02-15T20:06:19.704720Z","shell.execute_reply":"2024-02-15T20:06:20.781312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.to_csv('all_data_airpol_geo.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:20.783969Z","iopub.execute_input":"2024-02-15T20:06:20.784386Z","iopub.status.idle":"2024-02-15T20:06:23.007409Z","shell.execute_reply.started":"2024-02-15T20:06:20.784349Z","shell.execute_reply":"2024-02-15T20:06:23.006417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/widsdatathon2024-challenge1/training.csv')\ntest = pd.read_csv('/kaggle/input/widsdatathon2024-challenge1/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.008625Z","iopub.execute_input":"2024-02-15T20:06:23.009098Z","iopub.status.idle":"2024-02-15T20:06:23.363967Z","shell.execute_reply.started":"2024-02-15T20:06:23.009048Z","shell.execute_reply":"2024-02-15T20:06:23.363117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_type = pd.concat([ train.drop('DiagPeriodL90D', axis= 1), test], axis = 0)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.365069Z","iopub.execute_input":"2024-02-15T20:06:23.366196Z","iopub.status.idle":"2024-02-15T20:06:23.387050Z","shell.execute_reply.started":"2024-02-15T20:06:23.366132Z","shell.execute_reply":"2024-02-15T20:06:23.385721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_type.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.388539Z","iopub.execute_input":"2024-02-15T20:06:23.388909Z","iopub.status.idle":"2024-02-15T20:06:23.397633Z","shell.execute_reply.started":"2024-02-15T20:06:23.388877Z","shell.execute_reply":"2024-02-15T20:06:23.395933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['payer_type']= payer_type['payer_type']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.399044Z","iopub.execute_input":"2024-02-15T20:06:23.399463Z","iopub.status.idle":"2024-02-15T20:06:23.407805Z","shell.execute_reply.started":"2024-02-15T20:06:23.399426Z","shell.execute_reply":"2024-02-15T20:06:23.406554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Let's impute payer_type and patient_race","metadata":{}},{"cell_type":"code","source":"cat_cols = ['payer_type', 'patient_state', 'patient_zip3', 'breast_cancer_diagnosis_code', 'breast_cancer_diagnosis_desc', 'metastatic_cancer_diagnosis_code', 'Region', 'Division']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.410068Z","iopub.execute_input":"2024-02-15T20:06:23.410502Z","iopub.status.idle":"2024-02-15T20:06:23.419853Z","shell.execute_reply.started":"2024-02-15T20:06:23.410468Z","shell.execute_reply":"2024-02-15T20:06:23.418415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_cols:\n    all_data[col] = all_data[col].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.421927Z","iopub.execute_input":"2024-02-15T20:06:23.422439Z","iopub.status.idle":"2024-02-15T20:06:23.437751Z","shell.execute_reply.started":"2024-02-15T20:06:23.422402Z","shell.execute_reply":"2024-02-15T20:06:23.436299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss_payer = all_data['payer_type'].isna() #\ntrain_payer_data = all_data[~miss_payer] #complete data with payer_type\nmiss_payer_data = all_data[miss_payer] #data where payer type is missing and we have to impute it","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.439659Z","iopub.execute_input":"2024-02-15T20:06:23.440130Z","iopub.status.idle":"2024-02-15T20:06:23.456867Z","shell.execute_reply.started":"2024-02-15T20:06:23.440082Z","shell.execute_reply":"2024-02-15T20:06:23.455673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" all_data['payer_type'] = all_data['payer_type'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.459083Z","iopub.execute_input":"2024-02-15T20:06:23.460075Z","iopub.status.idle":"2024-02-15T20:06:23.465840Z","shell.execute_reply.started":"2024-02-15T20:06:23.460040Z","shell.execute_reply":"2024-02-15T20:06:23.464745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_payer_data.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.467283Z","iopub.execute_input":"2024-02-15T20:06:23.467679Z","iopub.status.idle":"2024-02-15T20:06:23.486500Z","shell.execute_reply.started":"2024-02-15T20:06:23.467647Z","shell.execute_reply":"2024-02-15T20:06:23.485270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.488489Z","iopub.execute_input":"2024-02-15T20:06:23.489438Z","iopub.status.idle":"2024-02-15T20:06:23.494461Z","shell.execute_reply.started":"2024-02-15T20:06:23.489397Z","shell.execute_reply":"2024-02-15T20:06:23.493618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import log_loss","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.496019Z","iopub.execute_input":"2024-02-15T20:06:23.496718Z","iopub.status.idle":"2024-02-15T20:06:23.506577Z","shell.execute_reply.started":"2024-02-15T20:06:23.496679Z","shell.execute_reply":"2024-02-15T20:06:23.505145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cat_cols = ['payer_type', 'patient_state', 'patient_zip3', 'breast_cancer_diagnosis_code', 'breast_cancer_diagnosis_desc', 'metastatic_cancer_diagnosis_code', 'Region', 'Division']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.508710Z","iopub.execute_input":"2024-02-15T20:06:23.509707Z","iopub.status.idle":"2024-02-15T20:06:23.521809Z","shell.execute_reply.started":"2024-02-15T20:06:23.509661Z","shell.execute_reply":"2024-02-15T20:06:23.520296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\n#for col in cat_cols:\n    #train_payer_data[col] = le.fit_transform(train_payer_data[col])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.524025Z","iopub.execute_input":"2024-02-15T20:06:23.524530Z","iopub.status.idle":"2024-02-15T20:06:23.533172Z","shell.execute_reply.started":"2024-02-15T20:06:23.524490Z","shell.execute_reply":"2024-02-15T20:06:23.531725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cat_cols_pred = ['patient_state', 'patient_zip3', 'breast_cancer_diagnosis_code', 'breast_cancer_diagnosis_desc', 'metastatic_cancer_diagnosis_code', 'Region', 'Division']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.534955Z","iopub.execute_input":"2024-02-15T20:06:23.535416Z","iopub.status.idle":"2024-02-15T20:06:23.544303Z","shell.execute_reply.started":"2024-02-15T20:06:23.535376Z","shell.execute_reply":"2024-02-15T20:06:23.543068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for col in cat_cols:\n#    miss_payer_data[col] = le.fit_transform(miss_payer_data[col])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.546246Z","iopub.execute_input":"2024-02-15T20:06:23.546717Z","iopub.status.idle":"2024-02-15T20:06:23.556536Z","shell.execute_reply.started":"2024-02-15T20:06:23.546677Z","shell.execute_reply":"2024-02-15T20:06:23.555455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.558599Z","iopub.execute_input":"2024-02-15T20:06:23.559098Z","iopub.status.idle":"2024-02-15T20:06:23.604092Z","shell.execute_reply.started":"2024-02-15T20:06:23.559045Z","shell.execute_reply":"2024-02-15T20:06:23.602618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_payer_data.drop(columns = ['patient_gender', 'index', 'patient_id', 'payer_type'])\ny = le.fit_transform(train_payer_data['payer_type'])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.606434Z","iopub.execute_input":"2024-02-15T20:06:23.606824Z","iopub.status.idle":"2024-02-15T20:06:23.625570Z","shell.execute_reply.started":"2024-02-15T20:06:23.606791Z","shell.execute_reply":"2024-02-15T20:06:23.624031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.4, stratify=y, shuffle=True, random_state=66)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.628399Z","iopub.execute_input":"2024-02-15T20:06:23.629574Z","iopub.status.idle":"2024-02-15T20:06:23.661020Z","shell.execute_reply.started":"2024-02-15T20:06:23.629527Z","shell.execute_reply":"2024-02-15T20:06:23.659807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n        'min_child_weight': [1, 5, 10],\n        'gamma': [0, 0.5, 1],\n        'subsample': [0.6, 0.8, 1.0],\n        'colsample_bytree': [0.6, 0.8, 1.0],\n        'max_depth': [3, 4, 5],\n        'learning_rate' : [0.001, 0.01],\n        'n_estimators' : [ 500, 1000]\n        }","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:06:23.663531Z","iopub.execute_input":"2024-02-15T20:06:23.663933Z","iopub.status.idle":"2024-02-15T20:06:23.670283Z","shell.execute_reply.started":"2024-02-15T20:06:23.663902Z","shell.execute_reply":"2024-02-15T20:06:23.669303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_new = XGBClassifier(objective='multi:softmax', enable_categorical=True)\nskf = StratifiedKFold(n_splits=5, shuffle = True, random_state = 66)\nrandom_new = RandomizedSearchCV(xgb_new, param_distributions=params, n_iter=80, scoring='neg_log_loss', n_jobs=4, cv=skf.split(X_train,y_train), verbose=0, random_state=66 )\nrandom_new.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:32:20.524422Z","iopub.execute_input":"2024-02-15T20:32:20.524920Z","iopub.status.idle":"2024-02-15T21:40:28.527858Z","shell.execute_reply.started":"2024-02-15T20:32:20.524883Z","shell.execute_reply":"2024-02-15T21:40:28.526455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:28.531030Z","iopub.execute_input":"2024-02-15T21:40:28.531849Z","iopub.status.idle":"2024-02-15T21:40:28.538629Z","shell.execute_reply.started":"2024-02-15T21:40:28.531773Z","shell.execute_reply":"2024-02-15T21:40:28.537120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:30:19.264180Z","iopub.execute_input":"2024-02-15T20:30:19.264639Z","iopub.status.idle":"2024-02-15T20:30:19.273018Z","shell.execute_reply.started":"2024-02-15T20:30:19.264604Z","shell.execute_reply":"2024-02-15T20:30:19.271838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_clf = XGBClassifier(**random_new.best_params_, enable_categorical=True)\npayer_clf.fit(X_train, y_train)\ntest_pred = payer_clf.predict(X_test)\nprint(confusion_matrix(y_test, test_pred))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:28.548409Z","iopub.execute_input":"2024-02-15T21:40:28.549538Z","iopub.status.idle":"2024-02-15T21:40:46.903883Z","shell.execute_reply.started":"2024-02-15T21:40:28.549498Z","shell.execute_reply":"2024-02-15T21:40:46.899945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['payer_type'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:46.905693Z","iopub.execute_input":"2024-02-15T21:40:46.906502Z","iopub.status.idle":"2024-02-15T21:40:46.920620Z","shell.execute_reply.started":"2024-02-15T21:40:46.906468Z","shell.execute_reply":"2024-02-15T21:40:46.919340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss_payer_data['payer_type'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:46.922964Z","iopub.execute_input":"2024-02-15T21:40:46.923895Z","iopub.status.idle":"2024-02-15T21:40:46.932313Z","shell.execute_reply.started":"2024-02-15T21:40:46.923819Z","shell.execute_reply":"2024-02-15T21:40:46.931256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_pred = payer_clf.predict(miss_payer_data.drop(columns = ['patient_gender', 'index', 'patient_id', 'payer_type']))\nprint(payer_pred)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.172573Z","iopub.execute_input":"2024-02-15T21:40:47.172934Z","iopub.status.idle":"2024-02-15T21:40:47.349779Z","shell.execute_reply.started":"2024-02-15T21:40:47.172904Z","shell.execute_reply":"2024-02-15T21:40:47.348418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_payer_type_index = payer_type[payer_type['payer_type'].isna()].index.tolist()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.351671Z","iopub.execute_input":"2024-02-15T21:40:47.352449Z","iopub.status.idle":"2024-02-15T21:40:47.366483Z","shell.execute_reply.started":"2024-02-15T21:40:47.352402Z","shell.execute_reply":"2024-02-15T21:40:47.364898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(missing_payer_type_index), len(payer_pred))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.374212Z","iopub.execute_input":"2024-02-15T21:40:47.375067Z","iopub.status.idle":"2024-02-15T21:40:47.381269Z","shell.execute_reply.started":"2024-02-15T21:40:47.375020Z","shell.execute_reply":"2024-02-15T21:40:47.379683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_payer_type_index[37]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.383637Z","iopub.execute_input":"2024-02-15T21:40:47.384105Z","iopub.status.idle":"2024-02-15T21:40:47.396303Z","shell.execute_reply.started":"2024-02-15T21:40:47.384065Z","shell.execute_reply":"2024-02-15T21:40:47.394785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_type.loc[296, 'payer_type']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.398626Z","iopub.execute_input":"2024-02-15T21:40:47.399121Z","iopub.status.idle":"2024-02-15T21:40:47.407401Z","shell.execute_reply.started":"2024-02-15T21:40:47.399080Z","shell.execute_reply":"2024-02-15T21:40:47.406485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.loc[296, 'payer_type']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.408400Z","iopub.execute_input":"2024-02-15T21:40:47.408718Z","iopub.status.idle":"2024-02-15T21:40:47.420885Z","shell.execute_reply.started":"2024-02-15T21:40:47.408688Z","shell.execute_reply":"2024-02-15T21:40:47.419398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['payer_type'].dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.422477Z","iopub.execute_input":"2024-02-15T21:40:47.422822Z","iopub.status.idle":"2024-02-15T21:40:47.432102Z","shell.execute_reply.started":"2024-02-15T21:40:47.422791Z","shell.execute_reply":"2024-02-15T21:40:47.430724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_cat = ['COMMERCIAL', 'MEDICAID', 'MEDICARE ADVANTAGE']\npayer_label = []\nfor label in payer_pred:\n    payer_label.append(payer_cat[label])\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.434114Z","iopub.execute_input":"2024-02-15T21:40:47.434463Z","iopub.status.idle":"2024-02-15T21:40:47.442713Z","shell.execute_reply.started":"2024-02-15T21:40:47.434433Z","shell.execute_reply":"2024-02-15T21:40:47.441568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for count, idx in enumerate(missing_payer_type_index):\n    all_data.loc[idx, 'payer_type'] = payer_label[count]\n    \n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.444637Z","iopub.execute_input":"2024-02-15T21:40:47.444974Z","iopub.status.idle":"2024-02-15T21:40:47.732474Z","shell.execute_reply.started":"2024-02-15T21:40:47.444944Z","shell.execute_reply":"2024-02-15T21:40:47.731221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.734293Z","iopub.execute_input":"2024-02-15T21:40:47.734685Z","iopub.status.idle":"2024-02-15T21:40:47.751130Z","shell.execute_reply.started":"2024-02-15T21:40:47.734652Z","shell.execute_reply":"2024-02-15T21:40:47.749822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.to_csv('all_data_airpol_geo_payer.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:47.753510Z","iopub.execute_input":"2024-02-15T21:40:47.753929Z","iopub.status.idle":"2024-02-15T21:40:50.033641Z","shell.execute_reply.started":"2024-02-15T21:40:47.753888Z","shell.execute_reply":"2024-02-15T21:40:50.032789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['patient_race'] = payer_type['patient_race']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.035053Z","iopub.execute_input":"2024-02-15T21:40:50.036437Z","iopub.status.idle":"2024-02-15T21:40:50.043235Z","shell.execute_reply.started":"2024-02-15T21:40:50.036384Z","shell.execute_reply":"2024-02-15T21:40:50.041907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.044781Z","iopub.execute_input":"2024-02-15T21:40:50.045987Z","iopub.status.idle":"2024-02-15T21:40:50.090135Z","shell.execute_reply.started":"2024-02-15T21:40:50.045944Z","shell.execute_reply":"2024-02-15T21:40:50.088688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss_race = all_data['patient_race'].isna()\ntrain_miss_race = all_data[~miss_race]\npred_miss_race = all_data[miss_race]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.091761Z","iopub.execute_input":"2024-02-15T21:40:50.092141Z","iopub.status.idle":"2024-02-15T21:40:50.110904Z","shell.execute_reply.started":"2024-02-15T21:40:50.092106Z","shell.execute_reply":"2024-02-15T21:40:50.109437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_miss_race.columns","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.112678Z","iopub.execute_input":"2024-02-15T21:40:50.113028Z","iopub.status.idle":"2024-02-15T21:40:50.122742Z","shell.execute_reply.started":"2024-02-15T21:40:50.112986Z","shell.execute_reply":"2024-02-15T21:40:50.121044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_miss_race.drop(columns=['index','patient_gender', 'patient_id', 'patient_race'])\ny = le.fit_transform(train_miss_race['patient_race'])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.124437Z","iopub.execute_input":"2024-02-15T21:40:50.124819Z","iopub.status.idle":"2024-02-15T21:40:50.143534Z","shell.execute_reply.started":"2024-02-15T21:40:50.124788Z","shell.execute_reply":"2024-02-15T21:40:50.141836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.4, stratify=y, shuffle=True, random_state=66)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.144934Z","iopub.execute_input":"2024-02-15T21:40:50.145436Z","iopub.status.idle":"2024-02-15T21:40:50.168094Z","shell.execute_reply.started":"2024-02-15T21:40:50.145394Z","shell.execute_reply":"2024-02-15T21:40:50.166947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_new = XGBClassifier(objective='multi:softmax', enable_categorical=True)\nskf = StratifiedKFold(n_splits=5, shuffle = True, random_state = 66)\nrandom_new = RandomizedSearchCV(xgb_new, param_distributions=params, n_iter=80, scoring='neg_log_loss', n_jobs=4, cv=skf.split(X_train,y_train), verbose=0, random_state=66 )\nrandom_new.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T21:40:50.169597Z","iopub.execute_input":"2024-02-15T21:40:50.170670Z","iopub.status.idle":"2024-02-15T22:51:34.047145Z","shell.execute_reply.started":"2024-02-15T21:40:50.170628Z","shell.execute_reply":"2024-02-15T22:51:34.045578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_clf = XGBClassifier(**random_new.best_params_, enable_categorical=True)\nrace_clf.fit(X_train, y_train)\ntest_pred = race_clf.predict(X_test)\nprint(accuracy_score(y_test, test_pred))\n\nprint(confusion_matrix(y_test, test_pred))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:51:34.049008Z","iopub.execute_input":"2024-02-15T22:51:34.049506Z","iopub.status.idle":"2024-02-15T22:51:46.272231Z","shell.execute_reply.started":"2024-02-15T22:51:34.049465Z","shell.execute_reply":"2024-02-15T22:51:46.270890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_pred = race_clf.predict(pred_miss_race.drop(columns = ['patient_gender', 'index', 'patient_id', 'patient_race']))\nprint(race_pred)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:51:46.274852Z","iopub.execute_input":"2024-02-15T22:51:46.275370Z","iopub.status.idle":"2024-02-15T22:51:46.739530Z","shell.execute_reply.started":"2024-02-15T22:51:46.275303Z","shell.execute_reply":"2024-02-15T22:51:46.738465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_race_index = payer_type[payer_type['patient_race'].isna()].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:51:46.746111Z","iopub.execute_input":"2024-02-15T22:51:46.747494Z","iopub.status.idle":"2024-02-15T22:51:46.764339Z","shell.execute_reply.started":"2024-02-15T22:51:46.747451Z","shell.execute_reply":"2024-02-15T22:51:46.762426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(missing_race_index), len(race_pred))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:51:46.766914Z","iopub.execute_input":"2024-02-15T22:51:46.767701Z","iopub.status.idle":"2024-02-15T22:51:46.777547Z","shell.execute_reply.started":"2024-02-15T22:51:46.767653Z","shell.execute_reply":"2024-02-15T22:51:46.775050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_type['patient_race'] = payer_type['patient_race'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:58:11.809645Z","iopub.execute_input":"2024-02-15T22:58:11.810713Z","iopub.status.idle":"2024-02-15T22:58:11.820706Z","shell.execute_reply.started":"2024-02-15T22:58:11.810672Z","shell.execute_reply":"2024-02-15T22:58:11.819788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payer_type['patient_race'].dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:58:51.469853Z","iopub.execute_input":"2024-02-15T22:58:51.470384Z","iopub.status.idle":"2024-02-15T22:58:51.479978Z","shell.execute_reply.started":"2024-02-15T22:58:51.470314Z","shell.execute_reply":"2024-02-15T22:58:51.478666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_cat = ['Asian', 'Black', 'Hispanic', 'Other', 'White']\nrace_label = []\nfor label in race_pred:\n    race_label.append(race_cat[label])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:59:16.718032Z","iopub.execute_input":"2024-02-15T22:59:16.719736Z","iopub.status.idle":"2024-02-15T22:59:16.732662Z","shell.execute_reply.started":"2024-02-15T22:59:16.719684Z","shell.execute_reply":"2024-02-15T22:59:16.731415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for count, idx in enumerate(missing_race_index):\n    all_data.loc[idx, 'patient_race'] = race_label[count]\n    \n","metadata":{"execution":{"iopub.status.busy":"2024-02-15T22:59:32.333843Z","iopub.execute_input":"2024-02-15T22:59:32.334256Z","iopub.status.idle":"2024-02-15T22:59:33.141736Z","shell.execute_reply.started":"2024-02-15T22:59:32.334224Z","shell.execute_reply":"2024-02-15T22:59:33.140420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.tail()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T23:00:50.145424Z","iopub.execute_input":"2024-02-15T23:00:50.145874Z","iopub.status.idle":"2024-02-15T23:00:50.183487Z","shell.execute_reply.started":"2024-02-15T23:00:50.145841Z","shell.execute_reply":"2024-02-15T23:00:50.182045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.to_csv('all_data_airpol_geo_payer_race.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T23:01:10.086010Z","iopub.execute_input":"2024-02-15T23:01:10.086478Z","iopub.status.idle":"2024-02-15T23:01:12.449219Z","shell.execute_reply.started":"2024-02-15T23:01:10.086442Z","shell.execute_reply":"2024-02-15T23:01:12.447809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sklearn.metrics.get_scorer_names()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T20:29:33.469818Z","iopub.status.idle":"2024-02-15T20:29:33.470368Z","shell.execute_reply.started":"2024-02-15T20:29:33.470081Z","shell.execute_reply":"2024-02-15T20:29:33.470104Z"},"trusted":true},"execution_count":null,"outputs":[]}]}