customer-churn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "findspark.init('/home/ubuntu/spark-2.1.1-bin-hadoop2.7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# Creating a SparkSession object - spark\n",
    "spark = SparkSession.builder.getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the data through spark.read.csv function\n",
    "train = spark.read.csv('tr2.csv', inferSchema= True, header = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+--------+-----------------+-----------------+---------------+------------------+-------------+----+---+------+--------------+-----------+----------+------------+-----------+------------------+-----------------+-------------------+----------------------+---------------------+----------------+\n",
      "|  msno|is_churn|payment_method_id|payment_plan_days|plan_list_price|actual_amount_paid|is_auto_renew|city| bd|gender|registered_via|trans_month|trans_year|expire_month|expire_year|registration_month|registration_year|membership_duration|autorenew_._not_cancel|registration_duration|reg_mem_duration|\n",
      "+------+--------+-----------------+-----------------+---------------+------------------+-------------+----+---+------+--------------+-----------+----------+------------+-----------+------------------+-----------------+-------------------+----------------------+---------------------+----------------+\n",
      "|179055|       0|               41|               30|            129|               129|            1|  22| 30|     1|             7|          8|      2015|           9|       2015|                 2|             2011|                 32|                     1|                 1673|            1641|\n",
      "|224971|       0|               41|               30|            129|               129|            1|   4| 32|     1|             7|          3|      2016|           4|       2016|                10|             2011|                 32|                     1|                 1646|            1614|\n",
      "| 24801|       0|               33|               30|            149|               149|            1|  13| 25|     1|             9|          2|      2017|           3|       2017|                 7|             2007|                 31|                     1|                 3531|            3500|\n",
      "|206468|       0|               41|               30|             99|                99|            1|  13| 21|     1|             7|         10|      2016|          11|       2016|                 7|             2015|                 31|                     1|                  489|             458|\n",
      "|241163|       0|               39|               31|            149|               149|            1|  15| 28|     1|             3|          2|      2015|           4|       2015|                 6|             2013|                 50|                     1|                  659|             609|\n",
      "|330799|       0|               41|               30|            149|               149|            1|  17| 23|     1|             7|          8|      2016|          10|       2016|                 4|             2012|                 31|                     1|                 1616|            1585|\n",
      "|331361|       0|               41|               30|            149|               149|            1|   4| 19|     1|             7|          2|      2015|           3|       2015|                 3|             2013|                 28|                     1|                  727|             699|\n",
      "| 80290|       0|               38|               30|            149|               149|            0|  13| 17|     0|             9|          6|      2016|           7|       2016|                12|             2014|                 30|                     0|                  579|             549|\n",
      "|124469|       0|               40|               30|            149|               149|            1|  13| 34|     0|             3|          8|      2016|           9|       2016|                12|             2014|                 31|                     1|                  662|             631|\n",
      "|193127|       0|               34|               30|            149|               149|            1|  13| 30|     1|             9|          7|      2016|           8|       2016|                 6|             2011|                 31|                     1|                 1909|            1878|\n",
      "|   994|       0|               31|               30|            149|               149|            1|   6| 60|     1|             3|          8|      2016|           9|       2016|                 8|             2012|                 30|                     1|                 1505|            1475|\n",
      "|321695|       0|               38|               30|            149|               149|            0|  12| 23|     0|             3|          7|      2016|           8|       2016|                 4|             2015|                 30|                     0|                  506|             476|\n",
      "|318324|       0|               31|               30|            149|               149|            1|  10| 42|     0|             3|          2|      2017|           3|       2017|                10|             2014|                 31|                     1|                  891|             860|\n",
      "|112070|       0|               41|               30|             99|                99|            1|   1|  0|     0|             7|          2|      2017|           3|       2017|                 8|             2015|                 28|                     1|                  578|             550|\n",
      "|287153|       0|               41|               30|            149|               149|            1|  14| 33|     1|             7|          2|      2016|           3|       2016|                 4|             2014|                 29|                     1|                  699|             670|\n",
      "|236518|       0|               41|               30|            149|               149|            1|   7| 18|     0|             7|         11|      2015|          12|       2015|                 3|             2011|                 31|                     1|                 1747|            1716|\n",
      "|139564|       0|               27|               30|            149|               149|            1|  22| 33|     1|             9|         11|      2016|          12|       2016|                10|             2013|                 29|                     1|                 1157|            1128|\n",
      "|389598|       0|               41|               30|             99|                99|            1|   4|  0|     1|             7|          6|      2016|           7|       2016|                12|             2013|                 30|                     1|                  956|             926|\n",
      "|103395|       0|               34|               30|            149|               149|            1|   4| 41|     1|             9|         10|      2016|          11|       2016|                10|             2009|                 30|                     1|                 2614|            2584|\n",
      "|216627|       0|               40|               30|            149|               149|            1|   4| 29|     1|             9|         11|      2015|          12|       2015|                 1|             2011|                 31|                     1|                 1805|            1774|\n",
      "+------+--------+-----------------+-----------------+---------------+------------------+-------------+----+---+------+--------------+-----------+----------+------------+-----------+------------------+-----------------+-------------------+----------------------+---------------------+----------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- msno: integer (nullable = true)\n",
      " |-- is_churn: integer (nullable = true)\n",
      " |-- payment_method_id: integer (nullable = true)\n",
      " |-- payment_plan_days: integer (nullable = true)\n",
      " |-- plan_list_price: integer (nullable = true)\n",
      " |-- actual_amount_paid: integer (nullable = true)\n",
      " |-- is_auto_renew: integer (nullable = true)\n",
      " |-- city: integer (nullable = true)\n",
      " |-- bd: integer (nullable = true)\n",
      " |-- gender: integer (nullable = true)\n",
      " |-- registered_via: integer (nullable = true)\n",
      " |-- trans_month: integer (nullable = true)\n",
      " |-- trans_year: integer (nullable = true)\n",
      " |-- expire_month: integer (nullable = true)\n",
      " |-- expire_year: integer (nullable = true)\n",
      " |-- registration_month: integer (nullable = true)\n",
      " |-- registration_year: integer (nullable = true)\n",
      " |-- membership_duration: integer (nullable = true)\n",
      " |-- autorenew_._not_cancel: integer (nullable = true)\n",
      " |-- registration_duration: integer (nullable = true)\n",
      " |-- reg_mem_duration: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "188"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting the input-output feature names\n",
    "assembler_bhai = VectorAssembler(inputCols = ['msno',\n",
    " 'payment_method_id',\n",
    " 'payment_plan_days',\n",
    " 'plan_list_price',\n",
    " 'actual_amount_paid',\n",
    " 'is_auto_renew',\n",
    " 'city',\n",
    " 'bd',\n",
    " 'gender',\n",
    " 'registered_via',\n",
    " 'trans_month',\n",
    " 'trans_year',\n",
    " 'expire_month',\n",
    " 'expire_year',\n",
    " 'registration_month',\n",
    " 'registration_year',\n",
    " 'membership_duration',\n",
    " 'registration_duration',\n",
    " 'reg_mem_duration'], outputCol = 'features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_bhai = assembler_bhai.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- msno: integer (nullable = true)\n",
      " |-- is_churn: integer (nullable = true)\n",
      " |-- payment_method_id: integer (nullable = true)\n",
      " |-- payment_plan_days: integer (nullable = true)\n",
      " |-- plan_list_price: integer (nullable = true)\n",
      " |-- actual_amount_paid: integer (nullable = true)\n",
      " |-- is_auto_renew: integer (nullable = true)\n",
      " |-- city: integer (nullable = true)\n",
      " |-- bd: integer (nullable = true)\n",
      " |-- gender: integer (nullable = true)\n",
      " |-- registered_via: integer (nullable = true)\n",
      " |-- trans_month: integer (nullable = true)\n",
      " |-- trans_year: integer (nullable = true)\n",
      " |-- expire_month: integer (nullable = true)\n",
      " |-- expire_year: integer (nullable = true)\n",
      " |-- registration_month: integer (nullable = true)\n",
      " |-- registration_year: integer (nullable = true)\n",
      " |-- membership_duration: integer (nullable = true)\n",
      " |-- autorenew_._not_cancel: integer (nullable = true)\n",
      " |-- registration_duration: integer (nullable = true)\n",
      " |-- reg_mem_duration: integer (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#This gives all the features including the target variable and features as a vector which only need to pas to the model\n",
    "output_bhai.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_data_bhai = output_bhai.select('features', 'is_churn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting the data into train and test\n",
    "train_is_churn, test_is_churn = final_data_bhai.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing the Logistic Regression function\n",
    "from pyspark.ml.classification import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assigning the target variable as is_churn\n",
    "lr_churn = LogisticRegression(labelCol = 'is_churn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitted_is_churn_model = lr_churn.fit(train_is_churn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing the evaluator function\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Evaluating the learned model to be evaluated on the test data\n",
    "pred_and_labels = fitted_is_churn_model.evaluate(test_is_churn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.ml.classification.BinaryLogisticRegressionSummary at 0x7f586206bf28>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_and_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------+--------------------+--------------------+----------+\n",
      "|            features|is_churn|       rawPrediction|         probability|prediction|\n",
      "+--------------------+--------+--------------------+--------------------+----------+\n",
      "|[733.0,39.0,30.0,...|       1|[-0.0645938833622...|[0.48385714160035...|       1.0|\n",
      "|[1248.0,38.0,30.0...|       1|[-0.0646676042740...|[0.48383873060540...|       1.0|\n",
      "|[3254.0,39.0,30.0...|       0|[-0.0645459789933...|[0.48386910521835...|       1.0|\n",
      "|[8905.0,41.0,30.0...|       1|[-0.0645784202072...|[0.48386100336049...|       1.0|\n",
      "|[10012.0,38.0,30....|       1|[-0.0646014152026...|[0.48385526060322...|       1.0|\n",
      "|[11173.0,34.0,30....|       0|[-0.0644323208247...|[0.48389749023798...|       1.0|\n",
      "|[16197.0,39.0,30....|       0|[-0.0644954197925...|[0.48388173187304...|       1.0|\n",
      "|[17533.0,41.0,30....|       1|[-0.0646247453167...|[0.48384943415795...|       1.0|\n",
      "|[30604.0,38.0,30....|       1|[-0.0646253859617...|[0.48384927416380...|       1.0|\n",
      "|[46405.0,34.0,30....|       0|[-0.0644682924228...|[0.48388850667076...|       1.0|\n",
      "|[51126.0,40.0,30....|       1|[-0.0645247102433...|[0.48387441687340...|       1.0|\n",
      "|[52896.0,36.0,30....|       0|[-0.0645624763498...|[0.48386498517301...|       1.0|\n",
      "|[66157.0,40.0,30....|       0|[-0.0644454081432...|[0.48389422180249...|       1.0|\n",
      "|[80290.0,38.0,30....|       0|[-0.0646252886626...|[0.48384929846320...|       1.0|\n",
      "|[80952.0,39.0,31....|       0|[-0.0645591931333...|[0.48386580512244...|       1.0|\n",
      "|[84894.0,41.0,30....|       0|[-0.0644825294931...|[0.48388495109969...|       1.0|\n",
      "|[86232.0,41.0,30....|       0|[-0.0644676609336...|[0.48388866437916...|       1.0|\n",
      "|[97570.0,41.0,30....|       1|[-0.0645168127356...|[0.48387638919697...|       1.0|\n",
      "|[100885.0,40.0,30...|       1|[-0.0644157583392...|[0.48390162656596...|       1.0|\n",
      "|[113084.0,38.0,30...|       1|[-0.0646005265851...|[0.48385548252598...|       1.0|\n",
      "+--------------------+--------+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_and_labels.predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "churn_eval = BinaryClassificationEvaluator(rawPredictionCol = 'prediction',labelCol='is_churn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "churn_eval_acc = BinaryClassificationEvaluator(rawPredictionCol = 'prediction',labelCol='is_churn',\n",
    "                                          metrics = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#auc of the model\n",
    "auc = churn_eval.evaluate(pred_and_labels.predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the RandomForestClassifier\n",
    "from pyspark.ml.classification import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the instant of RandomForestClassifier with labels and features\n",
    "rfc = RandomForestClassifier(labelCol = 'is_churn', featuresCol = 'features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training the RandomForest Classifier model on the training data set\n",
    "rfc_model = rfc.fit(train_is_churn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseVector(19, {0: 0.0809, 1: 0.0322, 2: 0.0021, 3: 0.0238, 4: 0.0626, 5: 0.0802, 6: 0.0512, 7: 0.0373, 8: 0.0155, 9: 0.0092, 10: 0.0391, 11: 0.0376, 12: 0.0109, 13: 0.0451, 14: 0.0395, 15: 0.0359, 16: 0.1842, 17: 0.1032, 18: 0.1094})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc_model.featureImportances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluating the learned model to be evaluated on the test data\n",
    "pred_and_labels_rfc = rfc_model.transform(test_is_churn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------+--------------------+\n",
      "|prediction|is_churn|            features|\n",
      "+----------+--------+--------------------+\n",
      "|       0.0|       1|[733.0,39.0,30.0,...|\n",
      "|       1.0|       1|[1248.0,38.0,30.0...|\n",
      "|       1.0|       0|[3254.0,39.0,30.0...|\n",
      "|       1.0|       1|[8905.0,41.0,30.0...|\n",
      "|       0.0|       1|[10012.0,38.0,30....|\n",
      "+----------+--------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_and_labels_rfc.select(\"prediction\", \"is_churn\", \"features\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[features: vector, is_churn: int, rawPrediction: vector, probability: vector, prediction: double]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_and_labels_rfc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computing test error\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol = 'is_churn', \n",
    "                                          predictionCol = 'prediction',\n",
    "                                          metricName = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = evaluator.evaluate(pred_and_labels_rfc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.59375"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_is_churn.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[features: vector, is_churn: int]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_is_churn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|            features|\n",
      "+--------------------+\n",
      "|[733.0,39.0,30.0,...|\n",
      "|[1248.0,38.0,30.0...|\n",
      "+--------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_is_churn.select('features').show(2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}