python_proj/modelling/notebooks/preprocessing/feature_transformation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature transformation\n",
    "\n",
    "Used to test the impact of various feature transformations on the dataset, and ultimately outputs a dataset with those transformations applied on it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:43:38.858606Z",
     "iopub.status.busy": "2024-02-21T16:43:38.858491Z",
     "iopub.status.idle": "2024-02-21T16:43:42.064856Z",
     "shell.execute_reply": "2024-02-21T16:43:42.064496Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_path='./data//final_data/dataset_subsampled.csv'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(df)=1216221.\n",
      "len(df.columns)=43\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Project Name</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>Closed At</th>\n",
       "      <th>ControlPullRequestLifeTimeInMinutes</th>\n",
       "      <th>ControlNumberOfCommitsInPullRequest</th>\n",
       "      <th>ControlIntraProjectPullRequestExperienceOfIntegrator</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSubmissionCount</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>WeightedEcosystemSecondOrderDegreeCentrality</th>\n",
       "      <th>WeightedIntraProjectSecondOrderDegreeCentrality</th>\n",
       "      <th>EcosystemLinkIntensity</th>\n",
       "      <th>IntraProjectLinkIntensity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.081164e+05</td>\n",
       "      <td>8.677039e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.685069e+04</td>\n",
       "      <td>4.210476e+00</td>\n",
       "      <td>4.014674e+01</td>\n",
       "      <td>9.869163e+00</td>\n",
       "      <td>5.027905e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>2.019806e-01</td>\n",
       "      <td>8.583325e-01</td>\n",
       "      <td>2.498338e+00</td>\n",
       "      <td>1.757198e+01</td>\n",
       "      <td>2.573036e-01</td>\n",
       "      <td>1.798150e+00</td>\n",
       "      <td>2.918379e+01</td>\n",
       "      <td>6.403151e+01</td>\n",
       "      <td>1.098749e+00</td>\n",
       "      <td>2.662755e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.243531e+05</td>\n",
       "      <td>5.806587e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.537950e+05</td>\n",
       "      <td>5.512267e+01</td>\n",
       "      <td>1.160820e+02</td>\n",
       "      <td>3.458581e+01</td>\n",
       "      <td>4.710578e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>1.710005e+00</td>\n",
       "      <td>9.254259e+00</td>\n",
       "      <td>7.565769e+00</td>\n",
       "      <td>6.527831e+01</td>\n",
       "      <td>2.921277e+00</td>\n",
       "      <td>1.891477e+01</td>\n",
       "      <td>1.216683e+03</td>\n",
       "      <td>3.492823e+03</td>\n",
       "      <td>9.274709e+00</td>\n",
       "      <td>2.778900e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.666667e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.542500e+05</td>\n",
       "      <td>3.224000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.708333e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.078700e+05</td>\n",
       "      <td>8.413000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.987833e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.100000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.180555e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.362318e+06</td>\n",
       "      <td>1.357000e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.997850e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>3.800000e+01</td>\n",
       "      <td>9.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.698931e+00</td>\n",
       "      <td>2.684865e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.475789e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.815970e+06</td>\n",
       "      <td>2.009400e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.489490e+06</td>\n",
       "      <td>1.000000e+04</td>\n",
       "      <td>4.271000e+03</td>\n",
       "      <td>1.699000e+03</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>3.730000e+02</td>\n",
       "      <td>9.270000e+02</td>\n",
       "      <td>2.332000e+03</td>\n",
       "      <td>7.630000e+03</td>\n",
       "      <td>7.410000e+02</td>\n",
       "      <td>2.764000e+03</td>\n",
       "      <td>5.534965e+05</td>\n",
       "      <td>2.524147e+06</td>\n",
       "      <td>9.048289e+02</td>\n",
       "      <td>5.249874e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Project Name  Submitter ID  PR Number  Closed At   \n",
       "count  1.216221e+06  1.216221e+06     1216221.0  1216221.0  1216221.0  \\\n",
       "mean   9.081164e+05  8.677039e+03           0.0        0.0        0.0   \n",
       "std    5.243531e+05  5.806587e+03           0.0        0.0        0.0   \n",
       "min    0.000000e+00  4.300000e+01           0.0        0.0        0.0   \n",
       "25%    4.542500e+05  3.224000e+03           0.0        0.0        0.0   \n",
       "50%    9.078700e+05  8.413000e+03           0.0        0.0        0.0   \n",
       "75%    1.362318e+06  1.357000e+04           0.0        0.0        0.0   \n",
       "max    1.815970e+06  2.009400e+04           0.0        0.0        0.0   \n",
       "\n",
       "       ControlPullRequestLifeTimeInMinutes   \n",
       "count                         1.216221e+06  \\\n",
       "mean                          3.685069e+04   \n",
       "std                           1.537950e+05   \n",
       "min                           1.666667e-02   \n",
       "25%                           3.708333e+01   \n",
       "50%                           8.987833e+02   \n",
       "75%                           7.997850e+03   \n",
       "max                           4.489490e+06   \n",
       "\n",
       "       ControlNumberOfCommitsInPullRequest   \n",
       "count                         1.216221e+06  \\\n",
       "mean                          4.210476e+00   \n",
       "std                           5.512267e+01   \n",
       "min                           0.000000e+00   \n",
       "25%                           1.000000e+00   \n",
       "50%                           1.000000e+00   \n",
       "75%                           3.000000e+00   \n",
       "max                           1.000000e+04   \n",
       "\n",
       "       ControlIntraProjectPullRequestExperienceOfIntegrator   \n",
       "count                                       1.216221e+06     \\\n",
       "mean                                        4.014674e+01      \n",
       "std                                         1.160820e+02      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         3.000000e+00      \n",
       "50%                                         1.100000e+01      \n",
       "75%                                         3.800000e+01      \n",
       "max                                         4.271000e+03      \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSubmissionCount   \n",
       "count                                     1.216221e+06  \\\n",
       "mean                                      9.869163e+00   \n",
       "std                                       3.458581e+01   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       1.000000e+00   \n",
       "75%                                       9.000000e+00   \n",
       "max                                       1.699000e+03   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate  ...   \n",
       "count                                 1.216221e+06  ...  \\\n",
       "mean                                  5.027905e-01  ...   \n",
       "std                                   4.710578e-01  ...   \n",
       "min                                   0.000000e+00  ...   \n",
       "25%                                   0.000000e+00  ...   \n",
       "50%                                   6.666667e-01  ...   \n",
       "75%                                   1.000000e+00  ...   \n",
       "max                                   1.000000e+00  ...   \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06           \\\n",
       "mean                                        2.019806e-01            \n",
       "std                                         1.710005e+00            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         0.000000e+00            \n",
       "max                                         3.730000e+02            \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06        \\\n",
       "mean                                        8.583325e-01         \n",
       "std                                         9.254259e+00         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         0.000000e+00         \n",
       "max                                         9.270000e+02         \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06              \\\n",
       "mean                                        2.498338e+00               \n",
       "std                                         7.565769e+00               \n",
       "min                                         0.000000e+00               \n",
       "25%                                         0.000000e+00               \n",
       "50%                                         0.000000e+00               \n",
       "75%                                         2.000000e+00               \n",
       "max                                         2.332000e+03               \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06           \\\n",
       "mean                                        1.757198e+01            \n",
       "std                                         6.527831e+01            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         1.000000e+00            \n",
       "75%                                         7.000000e+00            \n",
       "max                                         7.630000e+03            \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        2.573036e-01                    \n",
       "std                                         2.921277e+00                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         7.410000e+02                    \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        1.798150e+00                 \n",
       "std                                         1.891477e+01                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         2.764000e+03                 \n",
       "\n",
       "       WeightedEcosystemSecondOrderDegreeCentrality   \n",
       "count                                  1.216221e+06  \\\n",
       "mean                                   2.918379e+01   \n",
       "std                                    1.216683e+03   \n",
       "min                                    0.000000e+00   \n",
       "25%                                    0.000000e+00   \n",
       "50%                                    1.180555e-01   \n",
       "75%                                    2.698931e+00   \n",
       "max                                    5.534965e+05   \n",
       "\n",
       "       WeightedIntraProjectSecondOrderDegreeCentrality   \n",
       "count                                     1.216221e+06  \\\n",
       "mean                                      6.403151e+01   \n",
       "std                                       3.492823e+03   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       0.000000e+00   \n",
       "75%                                       2.684865e+00   \n",
       "max                                       2.524147e+06   \n",
       "\n",
       "       EcosystemLinkIntensity  IntraProjectLinkIntensity  \n",
       "count            1.216221e+06               1.216221e+06  \n",
       "mean             1.098749e+00               2.662755e+00  \n",
       "std              9.274709e+00               2.778900e+01  \n",
       "min              0.000000e+00               0.000000e+00  \n",
       "25%              0.000000e+00               0.000000e+00  \n",
       "50%              0.000000e+00               0.000000e+00  \n",
       "75%              0.000000e+00               2.475789e-01  \n",
       "max              9.048289e+02               5.249874e+03  \n",
       "\n",
       "[8 rows x 37 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import python_proj.utils.exp_utils as exp_utils\n",
    "\n",
    "input_data_file_name = \"dataset_subsampled\"\n",
    "\n",
    "data_path_format = exp_utils.BASE_PATH + \"/final_data/{data_file_name}.csv\"\n",
    "data_path = data_path_format.format(data_file_name=input_data_file_name)\n",
    "print(f\"{data_path=}\")\n",
    "\n",
    "meta_header_count = 5\n",
    "\n",
    "df: pd.DataFrame = pd.read_csv(filepath_or_buffer=data_path, header=0)\n",
    "\n",
    "print(f\"{len(df)=}.\")\n",
    "print(f\"{len(df.columns)=}\\n\")\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:43:42.066521Z",
     "iopub.status.busy": "2024-02-21T16:43:42.066394Z",
     "iopub.status.idle": "2024-02-21T16:43:42.157481Z",
     "shell.execute_reply": "2024-02-21T16:43:42.157089Z"
    }
   },
   "outputs": [],
   "source": [
    "predictors = df.columns[meta_header_count:]\n",
    "numeric_predictors = df[predictors].select_dtypes(include=\"number\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Log-transformation\n",
    "\n",
    "Applies one-off log-transform on count predictors (PR submission count, etc.)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:43:42.159810Z",
     "iopub.status.busy": "2024-02-21T16:43:42.159666Z",
     "iopub.status.idle": "2024-02-21T16:43:42.206879Z",
     "shell.execute_reply": "2024-02-21T16:43:42.206507Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count_predictors=['ControlPullRequestLifeTimeInMinutes', 'ControlNumberOfCommitsInPullRequest', 'ControlIntraProjectPullRequestExperienceOfIntegrator', 'IntraProjectSubmitterPullRequestSubmissionCount', 'IntraProjectSubmitterPullRequestCommentCount', 'EcosystemExperienceSubmitterPullRequestSubmissionCount', 'EcosystemExperienceSubmitterPullRequestCommentCount', 'DependencyEcosystemExperienceSubmitterPullRequestSubmissionCount', 'DependencyEcosystemExperienceSubmitterPullRequestCommentCount', 'NonDependencyEcosystemExperienceSubmitterPullRequestSubmissionCount', 'NonDependencyEcosystemExperienceSubmitterPullRequestCommentCount', 'InversedDependencyEcosystemExperienceSubmitterPullRequestSubmissionCount', 'InversedDependencyEcosystemExperienceSubmitterPullRequestCommentCount', 'IntraProjectSubmitterIssueSubmissionCount', 'IntraProjectSubmitterIssueCommentCount', 'EcosystemExperienceSubmitterIssueSubmissionCount', 'EcosystemExperienceSubmitterIssueCommentCount', 'DependencyEcosystemExperienceSubmitterIssueSubmissionCount', 'DependencyEcosystemExperienceSubmitterIssueCommentCount', 'NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount', 'NonDependencyEcosystemExperienceSubmitterIssueCommentCount', 'InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount', 'InversedDependencyEcosystemExperienceSubmitterIssueCommentCount', 'WeightedEcosystemSecondOrderDegreeCentrality', 'WeightedIntraProjectSecondOrderDegreeCentrality', 'EcosystemLinkIntensity', 'IntraProjectLinkIntensity']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import regex as re\n",
    "\n",
    "df_transformed = df.copy()\n",
    "\n",
    "sr_re = r\".*SuccessRate.*\"\n",
    "\n",
    "count_predictors = [\n",
    "    field for field in numeric_predictors.columns if not re.match(sr_re, field)\n",
    "]\n",
    "\n",
    "print(f\"{count_predictors=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:43:42.208448Z",
     "iopub.status.busy": "2024-02-21T16:43:42.208305Z",
     "iopub.status.idle": "2024-02-21T16:44:14.904082Z",
     "shell.execute_reply": "2024-02-21T16:44:14.903727Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Project Name</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>Closed At</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>EcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>ln(1 + DependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + DependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + NonDependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + WeightedEcosystemSecondOrderDegreeCentrality)</th>\n",
       "      <th>ln(1 + WeightedIntraProjectSecondOrderDegreeCentrality)</th>\n",
       "      <th>ln(1 + EcosystemLinkIntensity)</th>\n",
       "      <th>ln(1 + IntraProjectLinkIntensity)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.081164e+05</td>\n",
       "      <td>8.677039e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.027905e-01</td>\n",
       "      <td>4.027001e-01</td>\n",
       "      <td>9.860918e-02</td>\n",
       "      <td>3.749753e-01</td>\n",
       "      <td>5.920373e-02</td>\n",
       "      <td>...</td>\n",
       "      <td>6.788991e-02</td>\n",
       "      <td>1.347494e-01</td>\n",
       "      <td>6.172564e-01</td>\n",
       "      <td>1.201281e+00</td>\n",
       "      <td>6.301919e-02</td>\n",
       "      <td>1.387813e-01</td>\n",
       "      <td>8.459333e-01</td>\n",
       "      <td>8.907999e-01</td>\n",
       "      <td>1.491971e-01</td>\n",
       "      <td>3.827771e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.243531e+05</td>\n",
       "      <td>5.806587e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.710578e-01</td>\n",
       "      <td>4.523420e-01</td>\n",
       "      <td>2.908001e-01</td>\n",
       "      <td>4.479160e-01</td>\n",
       "      <td>2.284681e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>3.199921e-01</td>\n",
       "      <td>5.474352e-01</td>\n",
       "      <td>9.145723e-01</td>\n",
       "      <td>1.564078e+00</td>\n",
       "      <td>3.475801e-01</td>\n",
       "      <td>6.529677e-01</td>\n",
       "      <td>1.314167e+00</td>\n",
       "      <td>1.479044e+00</td>\n",
       "      <td>5.948867e-01</td>\n",
       "      <td>8.685340e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.542500e+05</td>\n",
       "      <td>3.224000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.078700e+05</td>\n",
       "      <td>8.413000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>6.931472e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.115910e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.362318e+06</td>\n",
       "      <td>1.357000e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.387755e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.230769e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.098612e+00</td>\n",
       "      <td>2.079442e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.308044e+00</td>\n",
       "      <td>1.304234e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.212048e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.815970e+06</td>\n",
       "      <td>2.009400e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>5.924256e+00</td>\n",
       "      <td>6.833032e+00</td>\n",
       "      <td>7.754910e+00</td>\n",
       "      <td>8.939974e+00</td>\n",
       "      <td>6.609349e+00</td>\n",
       "      <td>7.924796e+00</td>\n",
       "      <td>1.322401e+01</td>\n",
       "      <td>1.474141e+01</td>\n",
       "      <td>6.808850e+00</td>\n",
       "      <td>8.566150e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Project Name  Submitter ID  PR Number  Closed At   \n",
       "count  1.216221e+06  1.216221e+06     1216221.0  1216221.0  1216221.0  \\\n",
       "mean   9.081164e+05  8.677039e+03           0.0        0.0        0.0   \n",
       "std    5.243531e+05  5.806587e+03           0.0        0.0        0.0   \n",
       "min    0.000000e+00  4.300000e+01           0.0        0.0        0.0   \n",
       "25%    4.542500e+05  3.224000e+03           0.0        0.0        0.0   \n",
       "50%    9.078700e+05  8.413000e+03           0.0        0.0        0.0   \n",
       "75%    1.362318e+06  1.357000e+04           0.0        0.0        0.0   \n",
       "max    1.815970e+06  2.009400e+04           0.0        0.0        0.0   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate   \n",
       "count                                 1.216221e+06  \\\n",
       "mean                                  5.027905e-01   \n",
       "std                                   4.710578e-01   \n",
       "min                                   0.000000e+00   \n",
       "25%                                   0.000000e+00   \n",
       "50%                                   6.666667e-01   \n",
       "75%                                   1.000000e+00   \n",
       "max                                   1.000000e+00   \n",
       "\n",
       "       EcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06   \\\n",
       "mean                                        4.027001e-01    \n",
       "std                                         4.523420e-01    \n",
       "min                                         0.000000e+00    \n",
       "25%                                         0.000000e+00    \n",
       "50%                                         0.000000e+00    \n",
       "75%                                         9.387755e-01    \n",
       "max                                         1.000000e+00    \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06             \\\n",
       "mean                                        9.860918e-02              \n",
       "std                                         2.908001e-01              \n",
       "min                                         0.000000e+00              \n",
       "25%                                         0.000000e+00              \n",
       "50%                                         0.000000e+00              \n",
       "75%                                         0.000000e+00              \n",
       "max                                         1.000000e+00              \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        3.749753e-01                 \n",
       "std                                         4.479160e-01                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         9.230769e-01                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06                     \\\n",
       "mean                                        5.920373e-02                      \n",
       "std                                         2.284681e-01                      \n",
       "min                                         0.000000e+00                      \n",
       "25%                                         0.000000e+00                      \n",
       "50%                                         0.000000e+00                      \n",
       "75%                                         0.000000e+00                      \n",
       "max                                         1.000000e+00                      \n",
       "\n",
       "       ...   \n",
       "count  ...  \\\n",
       "mean   ...   \n",
       "std    ...   \n",
       "min    ...   \n",
       "25%    ...   \n",
       "50%    ...   \n",
       "75%    ...   \n",
       "max    ...   \n",
       "\n",
       "       ln(1 + DependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        6.788991e-02                    \n",
       "std                                         3.199921e-01                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         5.924256e+00                    \n",
       "\n",
       "       ln(1 + DependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        1.347494e-01                 \n",
       "std                                         5.474352e-01                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         6.833032e+00                 \n",
       "\n",
       "       ln(1 + NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                      \\\n",
       "mean                                        6.172564e-01                       \n",
       "std                                         9.145723e-01                       \n",
       "min                                         0.000000e+00                       \n",
       "25%                                         0.000000e+00                       \n",
       "50%                                         0.000000e+00                       \n",
       "75%                                         1.098612e+00                       \n",
       "max                                         7.754910e+00                       \n",
       "\n",
       "       ln(1 + NonDependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        1.201281e+00                    \n",
       "std                                         1.564078e+00                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         6.931472e-01                    \n",
       "75%                                         2.079442e+00                    \n",
       "max                                         8.939974e+00                    \n",
       "\n",
       "       ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                           \\\n",
       "mean                                        6.301919e-02                            \n",
       "std                                         3.475801e-01                            \n",
       "min                                         0.000000e+00                            \n",
       "25%                                         0.000000e+00                            \n",
       "50%                                         0.000000e+00                            \n",
       "75%                                         0.000000e+00                            \n",
       "max                                         6.609349e+00                            \n",
       "\n",
       "       ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                        \\\n",
       "mean                                        1.387813e-01                         \n",
       "std                                         6.529677e-01                         \n",
       "min                                         0.000000e+00                         \n",
       "25%                                         0.000000e+00                         \n",
       "50%                                         0.000000e+00                         \n",
       "75%                                         0.000000e+00                         \n",
       "max                                         7.924796e+00                         \n",
       "\n",
       "       ln(1 + WeightedEcosystemSecondOrderDegreeCentrality)   \n",
       "count                                       1.216221e+06     \\\n",
       "mean                                        8.459333e-01      \n",
       "std                                         1.314167e+00      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         0.000000e+00      \n",
       "50%                                         1.115910e-01      \n",
       "75%                                         1.308044e+00      \n",
       "max                                         1.322401e+01      \n",
       "\n",
       "       ln(1 + WeightedIntraProjectSecondOrderDegreeCentrality)   \n",
       "count                                       1.216221e+06        \\\n",
       "mean                                        8.907999e-01         \n",
       "std                                         1.479044e+00         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         1.304234e+00         \n",
       "max                                         1.474141e+01         \n",
       "\n",
       "       ln(1 + EcosystemLinkIntensity)  ln(1 + IntraProjectLinkIntensity)  \n",
       "count                    1.216221e+06                       1.216221e+06  \n",
       "mean                     1.491971e-01                       3.827771e-01  \n",
       "std                      5.948867e-01                       8.685340e-01  \n",
       "min                      0.000000e+00                       0.000000e+00  \n",
       "25%                      0.000000e+00                       0.000000e+00  \n",
       "50%                      0.000000e+00                       0.000000e+00  \n",
       "75%                      0.000000e+00                       2.212048e-01  \n",
       "max                      6.808850e+00                       8.566150e+00  \n",
       "\n",
       "[8 rows x 37 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Applies one-off log-transform\n",
    "for field in count_predictors:\n",
    "    new_key = f\"ln(1 + {field})\"\n",
    "    df_transformed[new_key] = df_transformed[field].apply(lambda x: np.log(1 + x))\n",
    "    df_transformed = df_transformed.drop(field, axis=1)\n",
    "\n",
    "df_transformed.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:44:14.905721Z",
     "iopub.status.busy": "2024-02-21T16:44:14.905565Z",
     "iopub.status.idle": "2024-02-21T16:44:15.093284Z",
     "shell.execute_reply": "2024-02-21T16:44:15.092879Z"
    }
   },
   "outputs": [],
   "source": [
    "# Collects the new numeric fields for the transformed dataframe.\n",
    "transformed_predictors = df_transformed.columns[meta_header_count:]\n",
    "transformed_numeric_predictors = df_transformed[transformed_predictors].select_dtypes(\n",
    "    include=\"number\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature scaling\n",
    "\n",
    "Applies min-max scaling on the features.\n",
    "This is applied to the transformed and the untransformed data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:44:15.095892Z",
     "iopub.status.busy": "2024-02-21T16:44:15.095748Z",
     "iopub.status.idle": "2024-02-21T16:44:15.098959Z",
     "shell.execute_reply": "2024-02-21T16:44:15.098633Z"
    }
   },
   "outputs": [],
   "source": [
    "def scale(_df: pd.DataFrame, scaled_fields: pd.Series):\n",
    "    scaled_df = _df.copy()\n",
    "\n",
    "    for feature in scaled_fields:\n",
    "        feature_min = scaled_df[feature].min()\n",
    "        feature_max = scaled_df[feature].max()\n",
    "        feature_delta = feature_max - feature_min\n",
    "\n",
    "        scaled_df[feature] = (\n",
    "            scaled_df[feature].subtract(feature_min).divide(feature_delta)\n",
    "        )\n",
    "\n",
    "    return scaled_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:44:15.101057Z",
     "iopub.status.busy": "2024-02-21T16:44:15.100942Z",
     "iopub.status.idle": "2024-02-21T16:44:16.200924Z",
     "shell.execute_reply": "2024-02-21T16:44:16.200552Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Project Name</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>Closed At</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>EcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>ln(1 + DependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + DependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + NonDependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + WeightedEcosystemSecondOrderDegreeCentrality)</th>\n",
       "      <th>ln(1 + WeightedIntraProjectSecondOrderDegreeCentrality)</th>\n",
       "      <th>ln(1 + EcosystemLinkIntensity)</th>\n",
       "      <th>ln(1 + IntraProjectLinkIntensity)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.081164e+05</td>\n",
       "      <td>8.677039e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.027905e-01</td>\n",
       "      <td>4.027001e-01</td>\n",
       "      <td>9.860918e-02</td>\n",
       "      <td>3.749753e-01</td>\n",
       "      <td>5.920373e-02</td>\n",
       "      <td>...</td>\n",
       "      <td>1.145965e-02</td>\n",
       "      <td>1.972030e-02</td>\n",
       "      <td>7.959556e-02</td>\n",
       "      <td>1.343719e-01</td>\n",
       "      <td>9.534855e-03</td>\n",
       "      <td>1.751229e-02</td>\n",
       "      <td>6.396949e-02</td>\n",
       "      <td>6.042839e-02</td>\n",
       "      <td>2.191224e-02</td>\n",
       "      <td>4.468485e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.243531e+05</td>\n",
       "      <td>5.806587e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.710578e-01</td>\n",
       "      <td>4.523420e-01</td>\n",
       "      <td>2.908001e-01</td>\n",
       "      <td>4.479160e-01</td>\n",
       "      <td>2.284681e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>5.401390e-02</td>\n",
       "      <td>8.011601e-02</td>\n",
       "      <td>1.179346e-01</td>\n",
       "      <td>1.749533e-01</td>\n",
       "      <td>5.258916e-02</td>\n",
       "      <td>8.239552e-02</td>\n",
       "      <td>9.937732e-02</td>\n",
       "      <td>1.003326e-01</td>\n",
       "      <td>8.736962e-02</td>\n",
       "      <td>1.013914e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.542500e+05</td>\n",
       "      <td>3.224000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.078700e+05</td>\n",
       "      <td>8.413000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>7.753347e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.438514e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.362318e+06</td>\n",
       "      <td>1.357000e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.387755e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.230769e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.416667e-01</td>\n",
       "      <td>2.326004e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.891430e-02</td>\n",
       "      <td>8.847414e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.582313e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.815970e+06</td>\n",
       "      <td>2.009400e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Project Name  Submitter ID  PR Number  Closed At   \n",
       "count  1.216221e+06  1.216221e+06     1216221.0  1216221.0  1216221.0  \\\n",
       "mean   9.081164e+05  8.677039e+03           0.0        0.0        0.0   \n",
       "std    5.243531e+05  5.806587e+03           0.0        0.0        0.0   \n",
       "min    0.000000e+00  4.300000e+01           0.0        0.0        0.0   \n",
       "25%    4.542500e+05  3.224000e+03           0.0        0.0        0.0   \n",
       "50%    9.078700e+05  8.413000e+03           0.0        0.0        0.0   \n",
       "75%    1.362318e+06  1.357000e+04           0.0        0.0        0.0   \n",
       "max    1.815970e+06  2.009400e+04           0.0        0.0        0.0   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate   \n",
       "count                                 1.216221e+06  \\\n",
       "mean                                  5.027905e-01   \n",
       "std                                   4.710578e-01   \n",
       "min                                   0.000000e+00   \n",
       "25%                                   0.000000e+00   \n",
       "50%                                   6.666667e-01   \n",
       "75%                                   1.000000e+00   \n",
       "max                                   1.000000e+00   \n",
       "\n",
       "       EcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06   \\\n",
       "mean                                        4.027001e-01    \n",
       "std                                         4.523420e-01    \n",
       "min                                         0.000000e+00    \n",
       "25%                                         0.000000e+00    \n",
       "50%                                         0.000000e+00    \n",
       "75%                                         9.387755e-01    \n",
       "max                                         1.000000e+00    \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06             \\\n",
       "mean                                        9.860918e-02              \n",
       "std                                         2.908001e-01              \n",
       "min                                         0.000000e+00              \n",
       "25%                                         0.000000e+00              \n",
       "50%                                         0.000000e+00              \n",
       "75%                                         0.000000e+00              \n",
       "max                                         1.000000e+00              \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        3.749753e-01                 \n",
       "std                                         4.479160e-01                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         9.230769e-01                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06                     \\\n",
       "mean                                        5.920373e-02                      \n",
       "std                                         2.284681e-01                      \n",
       "min                                         0.000000e+00                      \n",
       "25%                                         0.000000e+00                      \n",
       "50%                                         0.000000e+00                      \n",
       "75%                                         0.000000e+00                      \n",
       "max                                         1.000000e+00                      \n",
       "\n",
       "       ...   \n",
       "count  ...  \\\n",
       "mean   ...   \n",
       "std    ...   \n",
       "min    ...   \n",
       "25%    ...   \n",
       "50%    ...   \n",
       "75%    ...   \n",
       "max    ...   \n",
       "\n",
       "       ln(1 + DependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        1.145965e-02                    \n",
       "std                                         5.401390e-02                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         1.000000e+00                    \n",
       "\n",
       "       ln(1 + DependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        1.972030e-02                 \n",
       "std                                         8.011601e-02                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       ln(1 + NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                      \\\n",
       "mean                                        7.959556e-02                       \n",
       "std                                         1.179346e-01                       \n",
       "min                                         0.000000e+00                       \n",
       "25%                                         0.000000e+00                       \n",
       "50%                                         0.000000e+00                       \n",
       "75%                                         1.416667e-01                       \n",
       "max                                         1.000000e+00                       \n",
       "\n",
       "       ln(1 + NonDependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        1.343719e-01                    \n",
       "std                                         1.749533e-01                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         7.753347e-02                    \n",
       "75%                                         2.326004e-01                    \n",
       "max                                         1.000000e+00                    \n",
       "\n",
       "       ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                           \\\n",
       "mean                                        9.534855e-03                            \n",
       "std                                         5.258916e-02                            \n",
       "min                                         0.000000e+00                            \n",
       "25%                                         0.000000e+00                            \n",
       "50%                                         0.000000e+00                            \n",
       "75%                                         0.000000e+00                            \n",
       "max                                         1.000000e+00                            \n",
       "\n",
       "       ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                        \\\n",
       "mean                                        1.751229e-02                         \n",
       "std                                         8.239552e-02                         \n",
       "min                                         0.000000e+00                         \n",
       "25%                                         0.000000e+00                         \n",
       "50%                                         0.000000e+00                         \n",
       "75%                                         0.000000e+00                         \n",
       "max                                         1.000000e+00                         \n",
       "\n",
       "       ln(1 + WeightedEcosystemSecondOrderDegreeCentrality)   \n",
       "count                                       1.216221e+06     \\\n",
       "mean                                        6.396949e-02      \n",
       "std                                         9.937732e-02      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         0.000000e+00      \n",
       "50%                                         8.438514e-03      \n",
       "75%                                         9.891430e-02      \n",
       "max                                         1.000000e+00      \n",
       "\n",
       "       ln(1 + WeightedIntraProjectSecondOrderDegreeCentrality)   \n",
       "count                                       1.216221e+06        \\\n",
       "mean                                        6.042839e-02         \n",
       "std                                         1.003326e-01         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         8.847414e-02         \n",
       "max                                         1.000000e+00         \n",
       "\n",
       "       ln(1 + EcosystemLinkIntensity)  ln(1 + IntraProjectLinkIntensity)  \n",
       "count                    1.216221e+06                       1.216221e+06  \n",
       "mean                     2.191224e-02                       4.468485e-02  \n",
       "std                      8.736962e-02                       1.013914e-01  \n",
       "min                      0.000000e+00                       0.000000e+00  \n",
       "25%                      0.000000e+00                       0.000000e+00  \n",
       "50%                      0.000000e+00                       0.000000e+00  \n",
       "75%                      0.000000e+00                       2.582313e-02  \n",
       "max                      1.000000e+00                       1.000000e+00  \n",
       "\n",
       "[8 rows x 37 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_transformed = scale(df_transformed, transformed_numeric_predictors.columns)\n",
    "df_transformed.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:44:16.202709Z",
     "iopub.status.busy": "2024-02-21T16:44:16.202562Z",
     "iopub.status.idle": "2024-02-21T16:44:17.276299Z",
     "shell.execute_reply": "2024-02-21T16:44:17.275950Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Project Name</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>Closed At</th>\n",
       "      <th>ControlPullRequestLifeTimeInMinutes</th>\n",
       "      <th>ControlNumberOfCommitsInPullRequest</th>\n",
       "      <th>ControlIntraProjectPullRequestExperienceOfIntegrator</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSubmissionCount</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>WeightedEcosystemSecondOrderDegreeCentrality</th>\n",
       "      <th>WeightedIntraProjectSecondOrderDegreeCentrality</th>\n",
       "      <th>EcosystemLinkIntensity</th>\n",
       "      <th>IntraProjectLinkIntensity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.081164e+05</td>\n",
       "      <td>8.677039e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.208210e-03</td>\n",
       "      <td>4.210476e-04</td>\n",
       "      <td>9.399846e-03</td>\n",
       "      <td>5.808807e-03</td>\n",
       "      <td>5.027905e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>5.415028e-04</td>\n",
       "      <td>9.259250e-04</td>\n",
       "      <td>1.071328e-03</td>\n",
       "      <td>2.303012e-03</td>\n",
       "      <td>3.472383e-04</td>\n",
       "      <td>6.505608e-04</td>\n",
       "      <td>5.272624e-05</td>\n",
       "      <td>2.536758e-05</td>\n",
       "      <td>1.214317e-03</td>\n",
       "      <td>5.072037e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.243531e+05</td>\n",
       "      <td>5.806587e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.425667e-02</td>\n",
       "      <td>5.512267e-03</td>\n",
       "      <td>2.717911e-02</td>\n",
       "      <td>2.035657e-02</td>\n",
       "      <td>4.710578e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>4.584464e-03</td>\n",
       "      <td>9.983019e-03</td>\n",
       "      <td>3.244327e-03</td>\n",
       "      <td>8.555480e-03</td>\n",
       "      <td>3.942344e-03</td>\n",
       "      <td>6.843261e-03</td>\n",
       "      <td>2.198177e-03</td>\n",
       "      <td>1.383764e-03</td>\n",
       "      <td>1.025023e-02</td>\n",
       "      <td>5.293271e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.542500e+05</td>\n",
       "      <td>3.224000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.256321e-06</td>\n",
       "      <td>1.000000e-04</td>\n",
       "      <td>7.024116e-04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.078700e+05</td>\n",
       "      <td>8.413000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.001935e-04</td>\n",
       "      <td>1.000000e-04</td>\n",
       "      <td>2.575509e-03</td>\n",
       "      <td>5.885815e-04</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.310616e-04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.132904e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.362318e+06</td>\n",
       "      <td>1.357000e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.781457e-03</td>\n",
       "      <td>3.000000e-04</td>\n",
       "      <td>8.897214e-03</td>\n",
       "      <td>5.297234e-03</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.576329e-04</td>\n",
       "      <td>9.174312e-04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.876149e-06</td>\n",
       "      <td>1.063672e-06</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.715901e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.815970e+06</td>\n",
       "      <td>2.009400e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Project Name  Submitter ID  PR Number  Closed At   \n",
       "count  1.216221e+06  1.216221e+06     1216221.0  1216221.0  1216221.0  \\\n",
       "mean   9.081164e+05  8.677039e+03           0.0        0.0        0.0   \n",
       "std    5.243531e+05  5.806587e+03           0.0        0.0        0.0   \n",
       "min    0.000000e+00  4.300000e+01           0.0        0.0        0.0   \n",
       "25%    4.542500e+05  3.224000e+03           0.0        0.0        0.0   \n",
       "50%    9.078700e+05  8.413000e+03           0.0        0.0        0.0   \n",
       "75%    1.362318e+06  1.357000e+04           0.0        0.0        0.0   \n",
       "max    1.815970e+06  2.009400e+04           0.0        0.0        0.0   \n",
       "\n",
       "       ControlPullRequestLifeTimeInMinutes   \n",
       "count                         1.216221e+06  \\\n",
       "mean                          8.208210e-03   \n",
       "std                           3.425667e-02   \n",
       "min                           0.000000e+00   \n",
       "25%                           8.256321e-06   \n",
       "50%                           2.001935e-04   \n",
       "75%                           1.781457e-03   \n",
       "max                           1.000000e+00   \n",
       "\n",
       "       ControlNumberOfCommitsInPullRequest   \n",
       "count                         1.216221e+06  \\\n",
       "mean                          4.210476e-04   \n",
       "std                           5.512267e-03   \n",
       "min                           0.000000e+00   \n",
       "25%                           1.000000e-04   \n",
       "50%                           1.000000e-04   \n",
       "75%                           3.000000e-04   \n",
       "max                           1.000000e+00   \n",
       "\n",
       "       ControlIntraProjectPullRequestExperienceOfIntegrator   \n",
       "count                                       1.216221e+06     \\\n",
       "mean                                        9.399846e-03      \n",
       "std                                         2.717911e-02      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         7.024116e-04      \n",
       "50%                                         2.575509e-03      \n",
       "75%                                         8.897214e-03      \n",
       "max                                         1.000000e+00      \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSubmissionCount   \n",
       "count                                     1.216221e+06  \\\n",
       "mean                                      5.808807e-03   \n",
       "std                                       2.035657e-02   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       5.885815e-04   \n",
       "75%                                       5.297234e-03   \n",
       "max                                       1.000000e+00   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate  ...   \n",
       "count                                 1.216221e+06  ...  \\\n",
       "mean                                  5.027905e-01  ...   \n",
       "std                                   4.710578e-01  ...   \n",
       "min                                   0.000000e+00  ...   \n",
       "25%                                   0.000000e+00  ...   \n",
       "50%                                   6.666667e-01  ...   \n",
       "75%                                   1.000000e+00  ...   \n",
       "max                                   1.000000e+00  ...   \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06           \\\n",
       "mean                                        5.415028e-04            \n",
       "std                                         4.584464e-03            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         0.000000e+00            \n",
       "max                                         1.000000e+00            \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06        \\\n",
       "mean                                        9.259250e-04         \n",
       "std                                         9.983019e-03         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         0.000000e+00         \n",
       "max                                         1.000000e+00         \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06              \\\n",
       "mean                                        1.071328e-03               \n",
       "std                                         3.244327e-03               \n",
       "min                                         0.000000e+00               \n",
       "25%                                         0.000000e+00               \n",
       "50%                                         0.000000e+00               \n",
       "75%                                         8.576329e-04               \n",
       "max                                         1.000000e+00               \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06           \\\n",
       "mean                                        2.303012e-03            \n",
       "std                                         8.555480e-03            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         1.310616e-04            \n",
       "75%                                         9.174312e-04            \n",
       "max                                         1.000000e+00            \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        3.472383e-04                    \n",
       "std                                         3.942344e-03                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         1.000000e+00                    \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        6.505608e-04                 \n",
       "std                                         6.843261e-03                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       WeightedEcosystemSecondOrderDegreeCentrality   \n",
       "count                                  1.216221e+06  \\\n",
       "mean                                   5.272624e-05   \n",
       "std                                    2.198177e-03   \n",
       "min                                    0.000000e+00   \n",
       "25%                                    0.000000e+00   \n",
       "50%                                    2.132904e-07   \n",
       "75%                                    4.876149e-06   \n",
       "max                                    1.000000e+00   \n",
       "\n",
       "       WeightedIntraProjectSecondOrderDegreeCentrality   \n",
       "count                                     1.216221e+06  \\\n",
       "mean                                      2.536758e-05   \n",
       "std                                       1.383764e-03   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       0.000000e+00   \n",
       "75%                                       1.063672e-06   \n",
       "max                                       1.000000e+00   \n",
       "\n",
       "       EcosystemLinkIntensity  IntraProjectLinkIntensity  \n",
       "count            1.216221e+06               1.216221e+06  \n",
       "mean             1.214317e-03               5.072037e-04  \n",
       "std              1.025023e-02               5.293271e-03  \n",
       "min              0.000000e+00               0.000000e+00  \n",
       "25%              0.000000e+00               0.000000e+00  \n",
       "50%              0.000000e+00               0.000000e+00  \n",
       "75%              0.000000e+00               4.715901e-05  \n",
       "max              1.000000e+00               1.000000e+00  \n",
       "\n",
       "[8 rows x 37 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = scale(df, numeric_predictors.columns)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset safe to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:44:17.277866Z",
     "iopub.status.busy": "2024-02-21T16:44:17.277746Z",
     "iopub.status.idle": "2024-02-21T16:44:39.337718Z",
     "shell.execute_reply": "2024-02-21T16:44:39.337298Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Outputting to './data//final_data/dataset_transformed.csv'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Project Name</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>Closed At</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>EcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>ln(1 + DependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + DependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + NonDependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount)</th>\n",
       "      <th>ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueCommentCount)</th>\n",
       "      <th>ln(1 + WeightedEcosystemSecondOrderDegreeCentrality)</th>\n",
       "      <th>ln(1 + WeightedIntraProjectSecondOrderDegreeCentrality)</th>\n",
       "      <th>ln(1 + EcosystemLinkIntensity)</th>\n",
       "      <th>ln(1 + IntraProjectLinkIntensity)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.081164e+05</td>\n",
       "      <td>8.677039e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.027905e-01</td>\n",
       "      <td>4.027001e-01</td>\n",
       "      <td>9.860918e-02</td>\n",
       "      <td>3.749753e-01</td>\n",
       "      <td>5.920373e-02</td>\n",
       "      <td>...</td>\n",
       "      <td>1.145965e-02</td>\n",
       "      <td>1.972030e-02</td>\n",
       "      <td>7.959556e-02</td>\n",
       "      <td>1.343719e-01</td>\n",
       "      <td>9.534855e-03</td>\n",
       "      <td>1.751229e-02</td>\n",
       "      <td>6.396949e-02</td>\n",
       "      <td>6.042839e-02</td>\n",
       "      <td>2.191224e-02</td>\n",
       "      <td>4.468485e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.243531e+05</td>\n",
       "      <td>5.806587e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.710578e-01</td>\n",
       "      <td>4.523420e-01</td>\n",
       "      <td>2.908001e-01</td>\n",
       "      <td>4.479160e-01</td>\n",
       "      <td>2.284681e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>5.401390e-02</td>\n",
       "      <td>8.011601e-02</td>\n",
       "      <td>1.179346e-01</td>\n",
       "      <td>1.749533e-01</td>\n",
       "      <td>5.258916e-02</td>\n",
       "      <td>8.239552e-02</td>\n",
       "      <td>9.937732e-02</td>\n",
       "      <td>1.003326e-01</td>\n",
       "      <td>8.736962e-02</td>\n",
       "      <td>1.013914e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.542500e+05</td>\n",
       "      <td>3.224000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.078700e+05</td>\n",
       "      <td>8.413000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>7.753347e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.438514e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.362318e+06</td>\n",
       "      <td>1.357000e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.387755e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.230769e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.416667e-01</td>\n",
       "      <td>2.326004e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.891430e-02</td>\n",
       "      <td>8.847414e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.582313e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.815970e+06</td>\n",
       "      <td>2.009400e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Project Name  Submitter ID  PR Number  Closed At   \n",
       "count  1.216221e+06  1.216221e+06     1216221.0  1216221.0  1216221.0  \\\n",
       "mean   9.081164e+05  8.677039e+03           0.0        0.0        0.0   \n",
       "std    5.243531e+05  5.806587e+03           0.0        0.0        0.0   \n",
       "min    0.000000e+00  4.300000e+01           0.0        0.0        0.0   \n",
       "25%    4.542500e+05  3.224000e+03           0.0        0.0        0.0   \n",
       "50%    9.078700e+05  8.413000e+03           0.0        0.0        0.0   \n",
       "75%    1.362318e+06  1.357000e+04           0.0        0.0        0.0   \n",
       "max    1.815970e+06  2.009400e+04           0.0        0.0        0.0   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate   \n",
       "count                                 1.216221e+06  \\\n",
       "mean                                  5.027905e-01   \n",
       "std                                   4.710578e-01   \n",
       "min                                   0.000000e+00   \n",
       "25%                                   0.000000e+00   \n",
       "50%                                   6.666667e-01   \n",
       "75%                                   1.000000e+00   \n",
       "max                                   1.000000e+00   \n",
       "\n",
       "       EcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06   \\\n",
       "mean                                        4.027001e-01    \n",
       "std                                         4.523420e-01    \n",
       "min                                         0.000000e+00    \n",
       "25%                                         0.000000e+00    \n",
       "50%                                         0.000000e+00    \n",
       "75%                                         9.387755e-01    \n",
       "max                                         1.000000e+00    \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06             \\\n",
       "mean                                        9.860918e-02              \n",
       "std                                         2.908001e-01              \n",
       "min                                         0.000000e+00              \n",
       "25%                                         0.000000e+00              \n",
       "50%                                         0.000000e+00              \n",
       "75%                                         0.000000e+00              \n",
       "max                                         1.000000e+00              \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        3.749753e-01                 \n",
       "std                                         4.479160e-01                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         9.230769e-01                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterPullRequestSuccessRate   \n",
       "count                                       1.216221e+06                     \\\n",
       "mean                                        5.920373e-02                      \n",
       "std                                         2.284681e-01                      \n",
       "min                                         0.000000e+00                      \n",
       "25%                                         0.000000e+00                      \n",
       "50%                                         0.000000e+00                      \n",
       "75%                                         0.000000e+00                      \n",
       "max                                         1.000000e+00                      \n",
       "\n",
       "       ...   \n",
       "count  ...  \\\n",
       "mean   ...   \n",
       "std    ...   \n",
       "min    ...   \n",
       "25%    ...   \n",
       "50%    ...   \n",
       "75%    ...   \n",
       "max    ...   \n",
       "\n",
       "       ln(1 + DependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        1.145965e-02                    \n",
       "std                                         5.401390e-02                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         1.000000e+00                    \n",
       "\n",
       "       ln(1 + DependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        1.972030e-02                 \n",
       "std                                         8.011601e-02                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       ln(1 + NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                      \\\n",
       "mean                                        7.959556e-02                       \n",
       "std                                         1.179346e-01                       \n",
       "min                                         0.000000e+00                       \n",
       "25%                                         0.000000e+00                       \n",
       "50%                                         0.000000e+00                       \n",
       "75%                                         1.416667e-01                       \n",
       "max                                         1.000000e+00                       \n",
       "\n",
       "       ln(1 + NonDependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        1.343719e-01                    \n",
       "std                                         1.749533e-01                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         7.753347e-02                    \n",
       "75%                                         2.326004e-01                    \n",
       "max                                         1.000000e+00                    \n",
       "\n",
       "       ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount)   \n",
       "count                                       1.216221e+06                           \\\n",
       "mean                                        9.534855e-03                            \n",
       "std                                         5.258916e-02                            \n",
       "min                                         0.000000e+00                            \n",
       "25%                                         0.000000e+00                            \n",
       "50%                                         0.000000e+00                            \n",
       "75%                                         0.000000e+00                            \n",
       "max                                         1.000000e+00                            \n",
       "\n",
       "       ln(1 + InversedDependencyEcosystemExperienceSubmitterIssueCommentCount)   \n",
       "count                                       1.216221e+06                        \\\n",
       "mean                                        1.751229e-02                         \n",
       "std                                         8.239552e-02                         \n",
       "min                                         0.000000e+00                         \n",
       "25%                                         0.000000e+00                         \n",
       "50%                                         0.000000e+00                         \n",
       "75%                                         0.000000e+00                         \n",
       "max                                         1.000000e+00                         \n",
       "\n",
       "       ln(1 + WeightedEcosystemSecondOrderDegreeCentrality)   \n",
       "count                                       1.216221e+06     \\\n",
       "mean                                        6.396949e-02      \n",
       "std                                         9.937732e-02      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         0.000000e+00      \n",
       "50%                                         8.438514e-03      \n",
       "75%                                         9.891430e-02      \n",
       "max                                         1.000000e+00      \n",
       "\n",
       "       ln(1 + WeightedIntraProjectSecondOrderDegreeCentrality)   \n",
       "count                                       1.216221e+06        \\\n",
       "mean                                        6.042839e-02         \n",
       "std                                         1.003326e-01         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         8.847414e-02         \n",
       "max                                         1.000000e+00         \n",
       "\n",
       "       ln(1 + EcosystemLinkIntensity)  ln(1 + IntraProjectLinkIntensity)  \n",
       "count                    1.216221e+06                       1.216221e+06  \n",
       "mean                     2.191224e-02                       4.468485e-02  \n",
       "std                      8.736962e-02                       1.013914e-01  \n",
       "min                      0.000000e+00                       0.000000e+00  \n",
       "25%                      0.000000e+00                       0.000000e+00  \n",
       "50%                      0.000000e+00                       0.000000e+00  \n",
       "75%                      0.000000e+00                       2.582313e-02  \n",
       "max                      1.000000e+00                       1.000000e+00  \n",
       "\n",
       "[8 rows x 37 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_data_file_name = \"dataset_transformed\"\n",
    "output_path = data_path_format.format(data_file_name=output_data_file_name)\n",
    "print(f\"Outputting to '{output_path}'\")\n",
    "\n",
    "df_transformed.to_csv(output_path, index=False)\n",
    "\n",
    "df_transformed.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:44:39.339331Z",
     "iopub.status.busy": "2024-02-21T16:44:39.339202Z",
     "iopub.status.idle": "2024-02-21T16:45:01.700222Z",
     "shell.execute_reply": "2024-02-21T16:45:01.699870Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Outputting to './data//final_data/dataset_untransformed.csv'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Project Name</th>\n",
       "      <th>Submitter ID</th>\n",
       "      <th>PR Number</th>\n",
       "      <th>Closed At</th>\n",
       "      <th>ControlPullRequestLifeTimeInMinutes</th>\n",
       "      <th>ControlNumberOfCommitsInPullRequest</th>\n",
       "      <th>ControlIntraProjectPullRequestExperienceOfIntegrator</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSubmissionCount</th>\n",
       "      <th>IntraProjectSubmitterPullRequestSuccessRate</th>\n",
       "      <th>...</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>DependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>NonDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount</th>\n",
       "      <th>InversedDependencyEcosystemExperienceSubmitterIssueCommentCount</th>\n",
       "      <th>WeightedEcosystemSecondOrderDegreeCentrality</th>\n",
       "      <th>WeightedIntraProjectSecondOrderDegreeCentrality</th>\n",
       "      <th>EcosystemLinkIntensity</th>\n",
       "      <th>IntraProjectLinkIntensity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1216221.0</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "      <td>1.216221e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.081164e+05</td>\n",
       "      <td>8.677039e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.208210e-03</td>\n",
       "      <td>4.210476e-04</td>\n",
       "      <td>9.399846e-03</td>\n",
       "      <td>5.808807e-03</td>\n",
       "      <td>5.027905e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>5.415028e-04</td>\n",
       "      <td>9.259250e-04</td>\n",
       "      <td>1.071328e-03</td>\n",
       "      <td>2.303012e-03</td>\n",
       "      <td>3.472383e-04</td>\n",
       "      <td>6.505608e-04</td>\n",
       "      <td>5.272624e-05</td>\n",
       "      <td>2.536758e-05</td>\n",
       "      <td>1.214317e-03</td>\n",
       "      <td>5.072037e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.243531e+05</td>\n",
       "      <td>5.806587e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.425667e-02</td>\n",
       "      <td>5.512267e-03</td>\n",
       "      <td>2.717911e-02</td>\n",
       "      <td>2.035657e-02</td>\n",
       "      <td>4.710578e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>4.584464e-03</td>\n",
       "      <td>9.983019e-03</td>\n",
       "      <td>3.244327e-03</td>\n",
       "      <td>8.555480e-03</td>\n",
       "      <td>3.942344e-03</td>\n",
       "      <td>6.843261e-03</td>\n",
       "      <td>2.198177e-03</td>\n",
       "      <td>1.383764e-03</td>\n",
       "      <td>1.025023e-02</td>\n",
       "      <td>5.293271e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.542500e+05</td>\n",
       "      <td>3.224000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.256321e-06</td>\n",
       "      <td>1.000000e-04</td>\n",
       "      <td>7.024116e-04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.078700e+05</td>\n",
       "      <td>8.413000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.001935e-04</td>\n",
       "      <td>1.000000e-04</td>\n",
       "      <td>2.575509e-03</td>\n",
       "      <td>5.885815e-04</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.310616e-04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.132904e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.362318e+06</td>\n",
       "      <td>1.357000e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.781457e-03</td>\n",
       "      <td>3.000000e-04</td>\n",
       "      <td>8.897214e-03</td>\n",
       "      <td>5.297234e-03</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.576329e-04</td>\n",
       "      <td>9.174312e-04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.876149e-06</td>\n",
       "      <td>1.063672e-06</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.715901e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.815970e+06</td>\n",
       "      <td>2.009400e+04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID  Project Name  Submitter ID  PR Number  Closed At   \n",
       "count  1.216221e+06  1.216221e+06     1216221.0  1216221.0  1216221.0  \\\n",
       "mean   9.081164e+05  8.677039e+03           0.0        0.0        0.0   \n",
       "std    5.243531e+05  5.806587e+03           0.0        0.0        0.0   \n",
       "min    0.000000e+00  4.300000e+01           0.0        0.0        0.0   \n",
       "25%    4.542500e+05  3.224000e+03           0.0        0.0        0.0   \n",
       "50%    9.078700e+05  8.413000e+03           0.0        0.0        0.0   \n",
       "75%    1.362318e+06  1.357000e+04           0.0        0.0        0.0   \n",
       "max    1.815970e+06  2.009400e+04           0.0        0.0        0.0   \n",
       "\n",
       "       ControlPullRequestLifeTimeInMinutes   \n",
       "count                         1.216221e+06  \\\n",
       "mean                          8.208210e-03   \n",
       "std                           3.425667e-02   \n",
       "min                           0.000000e+00   \n",
       "25%                           8.256321e-06   \n",
       "50%                           2.001935e-04   \n",
       "75%                           1.781457e-03   \n",
       "max                           1.000000e+00   \n",
       "\n",
       "       ControlNumberOfCommitsInPullRequest   \n",
       "count                         1.216221e+06  \\\n",
       "mean                          4.210476e-04   \n",
       "std                           5.512267e-03   \n",
       "min                           0.000000e+00   \n",
       "25%                           1.000000e-04   \n",
       "50%                           1.000000e-04   \n",
       "75%                           3.000000e-04   \n",
       "max                           1.000000e+00   \n",
       "\n",
       "       ControlIntraProjectPullRequestExperienceOfIntegrator   \n",
       "count                                       1.216221e+06     \\\n",
       "mean                                        9.399846e-03      \n",
       "std                                         2.717911e-02      \n",
       "min                                         0.000000e+00      \n",
       "25%                                         7.024116e-04      \n",
       "50%                                         2.575509e-03      \n",
       "75%                                         8.897214e-03      \n",
       "max                                         1.000000e+00      \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSubmissionCount   \n",
       "count                                     1.216221e+06  \\\n",
       "mean                                      5.808807e-03   \n",
       "std                                       2.035657e-02   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       5.885815e-04   \n",
       "75%                                       5.297234e-03   \n",
       "max                                       1.000000e+00   \n",
       "\n",
       "       IntraProjectSubmitterPullRequestSuccessRate  ...   \n",
       "count                                 1.216221e+06  ...  \\\n",
       "mean                                  5.027905e-01  ...   \n",
       "std                                   4.710578e-01  ...   \n",
       "min                                   0.000000e+00  ...   \n",
       "25%                                   0.000000e+00  ...   \n",
       "50%                                   6.666667e-01  ...   \n",
       "75%                                   1.000000e+00  ...   \n",
       "max                                   1.000000e+00  ...   \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06           \\\n",
       "mean                                        5.415028e-04            \n",
       "std                                         4.584464e-03            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         0.000000e+00            \n",
       "75%                                         0.000000e+00            \n",
       "max                                         1.000000e+00            \n",
       "\n",
       "       DependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06        \\\n",
       "mean                                        9.259250e-04         \n",
       "std                                         9.983019e-03         \n",
       "min                                         0.000000e+00         \n",
       "25%                                         0.000000e+00         \n",
       "50%                                         0.000000e+00         \n",
       "75%                                         0.000000e+00         \n",
       "max                                         1.000000e+00         \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06              \\\n",
       "mean                                        1.071328e-03               \n",
       "std                                         3.244327e-03               \n",
       "min                                         0.000000e+00               \n",
       "25%                                         0.000000e+00               \n",
       "50%                                         0.000000e+00               \n",
       "75%                                         8.576329e-04               \n",
       "max                                         1.000000e+00               \n",
       "\n",
       "       NonDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06           \\\n",
       "mean                                        2.303012e-03            \n",
       "std                                         8.555480e-03            \n",
       "min                                         0.000000e+00            \n",
       "25%                                         0.000000e+00            \n",
       "50%                                         1.310616e-04            \n",
       "75%                                         9.174312e-04            \n",
       "max                                         1.000000e+00            \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueSubmissionCount   \n",
       "count                                       1.216221e+06                   \\\n",
       "mean                                        3.472383e-04                    \n",
       "std                                         3.942344e-03                    \n",
       "min                                         0.000000e+00                    \n",
       "25%                                         0.000000e+00                    \n",
       "50%                                         0.000000e+00                    \n",
       "75%                                         0.000000e+00                    \n",
       "max                                         1.000000e+00                    \n",
       "\n",
       "       InversedDependencyEcosystemExperienceSubmitterIssueCommentCount   \n",
       "count                                       1.216221e+06                \\\n",
       "mean                                        6.505608e-04                 \n",
       "std                                         6.843261e-03                 \n",
       "min                                         0.000000e+00                 \n",
       "25%                                         0.000000e+00                 \n",
       "50%                                         0.000000e+00                 \n",
       "75%                                         0.000000e+00                 \n",
       "max                                         1.000000e+00                 \n",
       "\n",
       "       WeightedEcosystemSecondOrderDegreeCentrality   \n",
       "count                                  1.216221e+06  \\\n",
       "mean                                   5.272624e-05   \n",
       "std                                    2.198177e-03   \n",
       "min                                    0.000000e+00   \n",
       "25%                                    0.000000e+00   \n",
       "50%                                    2.132904e-07   \n",
       "75%                                    4.876149e-06   \n",
       "max                                    1.000000e+00   \n",
       "\n",
       "       WeightedIntraProjectSecondOrderDegreeCentrality   \n",
       "count                                     1.216221e+06  \\\n",
       "mean                                      2.536758e-05   \n",
       "std                                       1.383764e-03   \n",
       "min                                       0.000000e+00   \n",
       "25%                                       0.000000e+00   \n",
       "50%                                       0.000000e+00   \n",
       "75%                                       1.063672e-06   \n",
       "max                                       1.000000e+00   \n",
       "\n",
       "       EcosystemLinkIntensity  IntraProjectLinkIntensity  \n",
       "count            1.216221e+06               1.216221e+06  \n",
       "mean             1.214317e-03               5.072037e-04  \n",
       "std              1.025023e-02               5.293271e-03  \n",
       "min              0.000000e+00               0.000000e+00  \n",
       "25%              0.000000e+00               0.000000e+00  \n",
       "50%              0.000000e+00               0.000000e+00  \n",
       "75%              0.000000e+00               4.715901e-05  \n",
       "max              1.000000e+00               1.000000e+00  \n",
       "\n",
       "[8 rows x 37 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_data_file_name = \"dataset_untransformed\"\n",
    "output_path = data_path_format.format(data_file_name=output_data_file_name)\n",
    "print(f\"Outputting to '{output_path}'\")\n",
    "\n",
    "df.to_csv(output_path, index=False)\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-21T16:45:01.701690Z",
     "iopub.status.busy": "2024-02-21T16:45:01.701551Z",
     "iopub.status.idle": "2024-02-21T16:45:01.703754Z",
     "shell.execute_reply": "2024-02-21T16:45:01.703477Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-02-21 16:45:01.702181\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "\n",
    "print(datetime.datetime.now())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}