ML_Pipeline_tutorial.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sankytanky100/End_to_End_ML_Project/blob/main/ML_Pipeline_tutorial.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler, OneHotEncoder\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "## Loading the dataset\n",
        "columns = [\"sex\",\"length\",\"diam\",\"height\",\"whole\",\"shucked\",\"viscera\",\"shell\",\"age\"]\n",
        "df = pd.read_csv(\"http://archive.ics.uci.edu/ml/machine-learning-databases/abalone/abalone.data\",names=columns)\n",
        "## Defining target and predictor variables\n",
        "y = df.age\n",
        "X = df.drop(columns=['age'])\n",
        "\n",
        "## Numerical columns:\n",
        "num_cols = X.select_dtypes(include=np.number).columns\n",
        "## Categorical columns\n",
        "cat_cols = X.select_dtypes(include=['object']).columns\n",
        "\n",
        "## Create some missing values\n",
        "for i in range(1000):\n",
        "    X.loc[np.random.choice(X.index),np.random.choice(X.columns)] = np.nan\n",
        "\n",
        "## Perform train-test split\n",
        "x_train, x_test, y_train, y_test = train_test_split(X,y, random_state=0, test_size=0.25)\n",
        "\n",
        "#####-------Imputation and Scaling: Code base to transform -----------------#####\n",
        "## Numerical training data\n",
        "x_train_num = x_train[num_cols]\n",
        "# Filling in missing values with mean on numeric features only\n",
        "x_train_fill_missing = x_train_num.fillna(x_train_num.mean())\n",
        "## Fitting standard scaler on x_train_fill_missing\n",
        "scale = StandardScaler().fit(x_train_fill_missing)\n",
        "## Scaling data after filling in missing values\n",
        "x_train_fill_missing_scale = scale.transform(x_train_fill_missing)\n",
        "## Same steps as above, but on the test set:\n",
        "x_test_fill_missing = x_test[num_cols].fillna(x_train_num.mean())\n",
        "x_test_fill_missing_scale = scale.transform(x_test_fill_missing)\n",
        "#####-------Imputation and Scaling: Code base to transform -----------------#####\n",
        "\n",
        "#1. Rewrite using Pipelines!\n",
        "pipeline = Pipeline([(\"imputer\",SimpleImputer(strategy='mean')), (\"scale\",StandardScaler())])\n",
        "\n",
        "#2. Fit pipeline on the test and compare results\n",
        "pipeline.fit(x_train[num_cols])\n",
        "x_transform = pipeline.transform(x_test[num_cols])\n",
        "\n",
        "# 3. Verify pipeline transform test set is the same by using np.array_equal()\n",
        "array_diff= np.array_equal(x_transform,x_test_fill_missing_scale)\n",
        "print(array_diff)\n",
        "\n",
        "#4. Change imputer strategy to median\n",
        "pipeline_median =Pipeline([(\"imputer\",SimpleImputer(strategy='median')), (\"scale\",StandardScaler())])\n",
        "pipeline_median.fit(x_train[num_cols])\n",
        "\n",
        "# 5 Compare results between the two pipelines\n",
        "x_transform_median = pipeline_median.transform(x_test[num_cols])\n",
        "new_array_diff = abs(x_transform-x_transform_median).sum()\n",
        "print(new_array_diff)"
      ],
      "metadata": {
        "id": "bLPlm2npB7mi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn import datasets\n",
        "from sklearn.model_selection import KFold, train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler, OneHotEncoder\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "\n",
        "columns = [\"sex\",\"length\",\"diam\",\"height\",\"whole\",\"shucked\",\"viscera\",\"shell\",\"age\"]\n",
        "df = pd.read_csv(\"http://archive.ics.uci.edu/ml/machine-learning-databases/abalone/abalone.data\",names=columns)\n",
        "\n",
        "y = df.age\n",
        "X=df.drop(columns=['age'])\n",
        "num_cols = X.select_dtypes(include=np.number).columns\n",
        "cat_cols = X.select_dtypes(include=['object']).columns\n",
        "#create some missing values\n",
        "for i in range(1000):\n",
        "    X.loc[np.random.choice(X.index),np.random.choice(X.columns)] = np.nan\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(X,y, random_state=0, test_size=0.25)\n",
        "x_train_cat = x_train[cat_cols]\n",
        "#fill missing values with mode on categorical features only\n",
        "x_train_fill_missing = x_train_cat.fillna(x_train_cat.mode().values[0][0])\n",
        "#apply one hot encoding on x_train_fill_missing\n",
        "ohe = OneHotEncoder(sparse=False, drop='first').fit(x_train_fill_missing)\n",
        "#transform data after filling in missing values\n",
        "x_train_fill_missing_ohe = ohe.transform(x_train_fill_missing)\n",
        "\n",
        "#Now want to do the same thing on the test set!\n",
        "x_test_fill_missing = x_test[cat_cols].fillna(x_train_cat.mode().values[0][0])\n",
        "x_test_fill_missing_ohe = ohe.transform(x_test_fill_missing)\n",
        "\n",
        "#1. Rewrite using Pipelines!\n",
        "pipeline = Pipeline([(\"imputer\",SimpleImputer(strategy='most_frequent')), (\"ohe\",OneHotEncoder(sparse=False, drop='first'))])\n",
        "\n",
        "\n",
        "#2. Fit the pipeline and transform the test data (categorical columns only!)\n",
        "pipeline.fit(x_train[cat_cols])\n",
        "x_transform = pipeline.transform(x_test[cat_cols])\n",
        "\n",
        "#3. Check if the two arrays are the same using np.array_equal()\n",
        "check_arrays = np.array_equal(x_transform,x_test_fill_missing_ohe)\n",
        "print('Are the arrays equal?')\n",
        "print(check_arrays)\n"
      ],
      "metadata": {
        "id": "jp73-xKHB6Oa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn import datasets\n",
        "from sklearn.model_selection import KFold, train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler, OneHotEncoder\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.impute import SimpleImputer\n",
        "\n",
        "columns = [\"sex\",\"length\",\"diam\",\"height\",\"whole\",\"shucked\",\"viscera\",\"shell\",\"age\"]\n",
        "df = pd.read_csv(\"http://archive.ics.uci.edu/ml/machine-learning-databases/abalone/abalone.data\",names=columns)\n",
        "\n",
        "y = df.age\n",
        "X=df.drop(columns=['age'])\n",
        "num_cols = X.select_dtypes(include=np.number).columns\n",
        "cat_cols = X.select_dtypes(include=['object']).columns\n",
        "#create some missing values\n",
        "for i in range(1000):\n",
        "    X.loc[np.random.choice(X.index),np.random.choice(X.columns)] = np.nan\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(X,y, random_state=0, test_size=0.25)\n",
        "\n",
        "#1. Create a pipeline `num_vals` to process numerical data\n",
        "\n",
        "num_vals = Pipeline([(\"imputer\",SimpleImputer()), (\"scale\",StandardScaler())])\n",
        "\n",
        "#2. Create a pipeline `cat_vals` to process categorical data\n",
        "cat_vals = Pipeline([(\"imputer\",SimpleImputer(strategy = 'most_frequent')), (\"ohe\",OneHotEncoder(drop = 'first', sparse = False))])\n",
        "\n",
        "\n",
        "#3. Create a column transformer, `preprocess` with the numerical and categorical pipelines\n",
        "preprocess = ColumnTransformer(\n",
        "    transformers=[\n",
        "        (\"num_preprocess\", num_vals, num_cols),\n",
        "        (\"cat_preprocess\", cat_vals, cat_cols)\n",
        "    ]\n",
        ")\n",
        "\n",
        "\n",
        "#4. Fit the preprocess transformer to training data\n",
        "preprocess.fit(x_train)\n",
        "#Transform the test data\n",
        "x_transform = preprocess.transform(x_test)\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "UWeZ3pezDSbF"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Welcome To Colab",
      "toc_visible": true,
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}