Machine Learning_Random Forest_EnergyGenerated.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Title:     SmartHome Dataset\n",
    "# Authors:   Aline J\n",
    "# Date:      01/06/2020\n",
    "# Goal:      Analyze and predict energy use efficiency \n",
    "#            in a smarthouse during a year under variable weather conditions.\n",
    "\n",
    "# Questions: \n",
    "# 1) What are the variables that control generated energy? \n",
    "# 2) What are the variables that control used energy? \n",
    "# 3) What are the variables that control energy efficiency?\n",
    "\n",
    "# Method. Unsupervised Machine learning Method, specifically random forest regression analysis.\n",
    "# Results: Used CleanData\n",
    "#         Several attempts to run the analysis were done at differen frequencies. \n",
    "#         Only Monthly frequency data resulted the highest accuracy (76%) vs Minute data (0.01%).  \n",
    "#         The minimum random state of 50, were enough to explain the variability in the dataset.  \n",
    "#         The same accuracy was yield when running random state of 1000.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: sklearn in c:\\programdata\\anaconda3\\lib\\site-packages (0.0)\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn in c:\\programdata\\anaconda3\\lib\\site-packages (from sklearn) (0.21.2)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in c:\\programdata\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (0.13.2)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.17.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (1.2.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.11.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn) (1.16.4)\n"
     ]
    }
   ],
   "source": [
    "### Begins Here ####\n",
    "# 1. Import dependencies\n",
    "# Update sklearn to prevent version mismatches\n",
    "!pip install sklearn --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: joblib in c:\\programdata\\anaconda3\\lib\\site-packages (0.13.2)\n"
     ]
    }
   ],
   "source": [
    "# install joblib. This will be used to save your model. \n",
    "# Restart your kernel after installing \n",
    "!pip install joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Other dependencies\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import matplotlib.pyplot as plt # plotting\n",
    "import numpy as np # linear algebra\n",
    "import os # accessing directory structure\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from scipy import stats\n",
    "import string as string\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 503911 rows and 32 columns\n"
     ]
    }
   ],
   "source": [
    "# 2. Import Raw data file into a data frame\n",
    "csv_path = \"Data/HomeC.csv\"\n",
    "my_data = pd.read_csv(csv_path,   parse_dates=True)\n",
    "home_dat = my_data.select_dtypes(exclude=['object'])\n",
    "nRow, nCol = my_data.shape\n",
    "print(f'There are {nRow} rows and {nCol} columns')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>use [kW]</th>\n",
       "      <th>gen [kW]</th>\n",
       "      <th>House overall [kW]</th>\n",
       "      <th>Dishwasher [kW]</th>\n",
       "      <th>Furnace 1 [kW]</th>\n",
       "      <th>Furnace 2 [kW]</th>\n",
       "      <th>Home office [kW]</th>\n",
       "      <th>Fridge [kW]</th>\n",
       "      <th>Wine cellar [kW]</th>\n",
       "      <th>Garage door [kW]</th>\n",
       "      <th>...</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>visibility</th>\n",
       "      <th>apparentTemperature</th>\n",
       "      <th>pressure</th>\n",
       "      <th>windSpeed</th>\n",
       "      <th>windBearing</th>\n",
       "      <th>precipIntensity</th>\n",
       "      <th>dewPoint</th>\n",
       "      <th>precipProbability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>503906</th>\n",
       "      <td>1.599333</td>\n",
       "      <td>0.003233</td>\n",
       "      <td>1.599333</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.104017</td>\n",
       "      <td>0.625033</td>\n",
       "      <td>0.041750</td>\n",
       "      <td>0.005233</td>\n",
       "      <td>0.008433</td>\n",
       "      <td>0.013433</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503907</th>\n",
       "      <td>1.924267</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.924267</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.422383</td>\n",
       "      <td>0.637733</td>\n",
       "      <td>0.042033</td>\n",
       "      <td>0.004983</td>\n",
       "      <td>0.008467</td>\n",
       "      <td>0.012933</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503908</th>\n",
       "      <td>1.978200</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.978200</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.495667</td>\n",
       "      <td>0.620367</td>\n",
       "      <td>0.042100</td>\n",
       "      <td>0.005333</td>\n",
       "      <td>0.008233</td>\n",
       "      <td>0.012817</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503909</th>\n",
       "      <td>1.990950</td>\n",
       "      <td>0.003233</td>\n",
       "      <td>1.990950</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.494700</td>\n",
       "      <td>0.634133</td>\n",
       "      <td>0.042100</td>\n",
       "      <td>0.004917</td>\n",
       "      <td>0.008133</td>\n",
       "      <td>0.012833</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503910</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        use [kW]  gen [kW]  House overall [kW]  Dishwasher [kW]  \\\n",
       "503906  1.599333  0.003233            1.599333         0.000050   \n",
       "503907  1.924267  0.003217            1.924267         0.000033   \n",
       "503908  1.978200  0.003217            1.978200         0.000050   \n",
       "503909  1.990950  0.003233            1.990950         0.000050   \n",
       "503910       NaN       NaN                 NaN              NaN   \n",
       "\n",
       "        Furnace 1 [kW]  Furnace 2 [kW]  Home office [kW]  Fridge [kW]  \\\n",
       "503906        0.104017        0.625033          0.041750     0.005233   \n",
       "503907        0.422383        0.637733          0.042033     0.004983   \n",
       "503908        0.495667        0.620367          0.042100     0.005333   \n",
       "503909        0.494700        0.634133          0.042100     0.004917   \n",
       "503910             NaN             NaN               NaN          NaN   \n",
       "\n",
       "        Wine cellar [kW]  Garage door [kW]  ...  temperature  humidity  \\\n",
       "503906          0.008433          0.013433  ...        35.12      0.86   \n",
       "503907          0.008467          0.012933  ...        35.12      0.86   \n",
       "503908          0.008233          0.012817  ...        35.12      0.86   \n",
       "503909          0.008133          0.012833  ...        35.12      0.86   \n",
       "503910               NaN               NaN  ...          NaN       NaN   \n",
       "\n",
       "        visibility  apparentTemperature  pressure  windSpeed  windBearing  \\\n",
       "503906        8.74                29.45   1011.49       6.72        186.0   \n",
       "503907        8.74                29.45   1011.49       6.72        186.0   \n",
       "503908        8.74                29.45   1011.49       6.72        186.0   \n",
       "503909        8.74                29.45   1011.49       6.72        186.0   \n",
       "503910         NaN                  NaN       NaN        NaN          NaN   \n",
       "\n",
       "        precipIntensity  dewPoint  precipProbability  \n",
       "503906           0.0101     31.27               0.51  \n",
       "503907           0.0101     31.27               0.51  \n",
       "503908           0.0101     31.27               0.51  \n",
       "503909           0.0101     31.27               0.51  \n",
       "503910              NaN       NaN                NaN  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "home_dat.head(5)\n",
    "home_dat.tail(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_index = pd.date_range('2016-01-01 05:00', periods=503911,  freq='min')  \n",
    "time_index = pd.DatetimeIndex(time_index)\n",
    "home_dat = home_dat.set_index(time_index)\n",
    "# Check that a dataset has been uploaded into my_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>use [kW]</th>\n",
       "      <th>gen [kW]</th>\n",
       "      <th>House overall [kW]</th>\n",
       "      <th>Dishwasher [kW]</th>\n",
       "      <th>Furnace 1 [kW]</th>\n",
       "      <th>Furnace 2 [kW]</th>\n",
       "      <th>Home office [kW]</th>\n",
       "      <th>Fridge [kW]</th>\n",
       "      <th>Wine cellar [kW]</th>\n",
       "      <th>Garage door [kW]</th>\n",
       "      <th>...</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>visibility</th>\n",
       "      <th>apparentTemperature</th>\n",
       "      <th>pressure</th>\n",
       "      <th>windSpeed</th>\n",
       "      <th>windBearing</th>\n",
       "      <th>precipIntensity</th>\n",
       "      <th>dewPoint</th>\n",
       "      <th>precipProbability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:26:00</th>\n",
       "      <td>1.599333</td>\n",
       "      <td>0.003233</td>\n",
       "      <td>1.599333</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.104017</td>\n",
       "      <td>0.625033</td>\n",
       "      <td>0.041750</td>\n",
       "      <td>0.005233</td>\n",
       "      <td>0.008433</td>\n",
       "      <td>0.013433</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:27:00</th>\n",
       "      <td>1.924267</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.924267</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.422383</td>\n",
       "      <td>0.637733</td>\n",
       "      <td>0.042033</td>\n",
       "      <td>0.004983</td>\n",
       "      <td>0.008467</td>\n",
       "      <td>0.012933</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:28:00</th>\n",
       "      <td>1.978200</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.978200</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.495667</td>\n",
       "      <td>0.620367</td>\n",
       "      <td>0.042100</td>\n",
       "      <td>0.005333</td>\n",
       "      <td>0.008233</td>\n",
       "      <td>0.012817</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:29:00</th>\n",
       "      <td>1.990950</td>\n",
       "      <td>0.003233</td>\n",
       "      <td>1.990950</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.494700</td>\n",
       "      <td>0.634133</td>\n",
       "      <td>0.042100</td>\n",
       "      <td>0.004917</td>\n",
       "      <td>0.008133</td>\n",
       "      <td>0.012833</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:30:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     use [kW]  gen [kW]  House overall [kW]  Dishwasher [kW]  \\\n",
       "2016-12-16 03:26:00  1.599333  0.003233            1.599333         0.000050   \n",
       "2016-12-16 03:27:00  1.924267  0.003217            1.924267         0.000033   \n",
       "2016-12-16 03:28:00  1.978200  0.003217            1.978200         0.000050   \n",
       "2016-12-16 03:29:00  1.990950  0.003233            1.990950         0.000050   \n",
       "2016-12-16 03:30:00       NaN       NaN                 NaN              NaN   \n",
       "\n",
       "                     Furnace 1 [kW]  Furnace 2 [kW]  Home office [kW]  \\\n",
       "2016-12-16 03:26:00        0.104017        0.625033          0.041750   \n",
       "2016-12-16 03:27:00        0.422383        0.637733          0.042033   \n",
       "2016-12-16 03:28:00        0.495667        0.620367          0.042100   \n",
       "2016-12-16 03:29:00        0.494700        0.634133          0.042100   \n",
       "2016-12-16 03:30:00             NaN             NaN               NaN   \n",
       "\n",
       "                     Fridge [kW]  Wine cellar [kW]  Garage door [kW]  ...  \\\n",
       "2016-12-16 03:26:00     0.005233          0.008433          0.013433  ...   \n",
       "2016-12-16 03:27:00     0.004983          0.008467          0.012933  ...   \n",
       "2016-12-16 03:28:00     0.005333          0.008233          0.012817  ...   \n",
       "2016-12-16 03:29:00     0.004917          0.008133          0.012833  ...   \n",
       "2016-12-16 03:30:00          NaN               NaN               NaN  ...   \n",
       "\n",
       "                     temperature  humidity  visibility  apparentTemperature  \\\n",
       "2016-12-16 03:26:00        35.12      0.86        8.74                29.45   \n",
       "2016-12-16 03:27:00        35.12      0.86        8.74                29.45   \n",
       "2016-12-16 03:28:00        35.12      0.86        8.74                29.45   \n",
       "2016-12-16 03:29:00        35.12      0.86        8.74                29.45   \n",
       "2016-12-16 03:30:00          NaN       NaN         NaN                  NaN   \n",
       "\n",
       "                     pressure  windSpeed  windBearing  precipIntensity  \\\n",
       "2016-12-16 03:26:00   1011.49       6.72        186.0           0.0101   \n",
       "2016-12-16 03:27:00   1011.49       6.72        186.0           0.0101   \n",
       "2016-12-16 03:28:00   1011.49       6.72        186.0           0.0101   \n",
       "2016-12-16 03:29:00   1011.49       6.72        186.0           0.0101   \n",
       "2016-12-16 03:30:00       NaN        NaN          NaN              NaN   \n",
       "\n",
       "                     dewPoint  precipProbability  \n",
       "2016-12-16 03:26:00     31.27               0.51  \n",
       "2016-12-16 03:27:00     31.27               0.51  \n",
       "2016-12-16 03:28:00     31.27               0.51  \n",
       "2016-12-16 03:29:00     31.27               0.51  \n",
       "2016-12-16 03:30:00       NaN                NaN  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "home_dat.head(5)\n",
    "home_dat.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['use', 'gen', 'House overall', 'Dishwasher', 'Furnace 1', 'Furnace 2',\n",
       "       'Home office', 'Fridge', 'Wine cellar', 'Garage door', 'Kitchen 12',\n",
       "       'Kitchen 14', 'Kitchen 38', 'Barn', 'Well', 'Microwave', 'Living room',\n",
       "       'Solar', 'temperature', 'humidity', 'visibility', 'apparentTemperature',\n",
       "       'pressure', 'windSpeed', 'windBearing', 'precipIntensity', 'dewPoint',\n",
       "       'precipProbability'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "home_dat.columns = [col.replace(' [kW]', '') for col in home_dat.columns]\n",
    "home_dat.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_dat = home_dat.dropna(axis='columns', how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>use</th>\n",
       "      <th>gen</th>\n",
       "      <th>House overall</th>\n",
       "      <th>Dishwasher</th>\n",
       "      <th>Furnace 1</th>\n",
       "      <th>Furnace 2</th>\n",
       "      <th>Home office</th>\n",
       "      <th>Fridge</th>\n",
       "      <th>Wine cellar</th>\n",
       "      <th>Garage door</th>\n",
       "      <th>...</th>\n",
       "      <th>temperature</th>\n",
       "      <th>humidity</th>\n",
       "      <th>visibility</th>\n",
       "      <th>apparentTemperature</th>\n",
       "      <th>pressure</th>\n",
       "      <th>windSpeed</th>\n",
       "      <th>windBearing</th>\n",
       "      <th>precipIntensity</th>\n",
       "      <th>dewPoint</th>\n",
       "      <th>precipProbability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:27:00</th>\n",
       "      <td>1.924267</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.924267</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.422383</td>\n",
       "      <td>0.637733</td>\n",
       "      <td>0.042033</td>\n",
       "      <td>0.004983</td>\n",
       "      <td>0.008467</td>\n",
       "      <td>0.012933</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:28:00</th>\n",
       "      <td>1.978200</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.978200</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.495667</td>\n",
       "      <td>0.620367</td>\n",
       "      <td>0.042100</td>\n",
       "      <td>0.005333</td>\n",
       "      <td>0.008233</td>\n",
       "      <td>0.012817</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:29:00</th>\n",
       "      <td>1.990950</td>\n",
       "      <td>0.003233</td>\n",
       "      <td>1.990950</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.494700</td>\n",
       "      <td>0.634133</td>\n",
       "      <td>0.042100</td>\n",
       "      <td>0.004917</td>\n",
       "      <td>0.008133</td>\n",
       "      <td>0.012833</td>\n",
       "      <td>...</td>\n",
       "      <td>35.12</td>\n",
       "      <td>0.86</td>\n",
       "      <td>8.74</td>\n",
       "      <td>29.45</td>\n",
       "      <td>1011.49</td>\n",
       "      <td>6.72</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0101</td>\n",
       "      <td>31.27</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          use       gen  House overall  Dishwasher  Furnace 1  \\\n",
       "2016-12-16 03:27:00  1.924267  0.003217       1.924267    0.000033   0.422383   \n",
       "2016-12-16 03:28:00  1.978200  0.003217       1.978200    0.000050   0.495667   \n",
       "2016-12-16 03:29:00  1.990950  0.003233       1.990950    0.000050   0.494700   \n",
       "\n",
       "                     Furnace 2  Home office    Fridge  Wine cellar  \\\n",
       "2016-12-16 03:27:00   0.637733     0.042033  0.004983     0.008467   \n",
       "2016-12-16 03:28:00   0.620367     0.042100  0.005333     0.008233   \n",
       "2016-12-16 03:29:00   0.634133     0.042100  0.004917     0.008133   \n",
       "\n",
       "                     Garage door  ...  temperature  humidity  visibility  \\\n",
       "2016-12-16 03:27:00     0.012933  ...        35.12      0.86        8.74   \n",
       "2016-12-16 03:28:00     0.012817  ...        35.12      0.86        8.74   \n",
       "2016-12-16 03:29:00     0.012833  ...        35.12      0.86        8.74   \n",
       "\n",
       "                     apparentTemperature  pressure  windSpeed  windBearing  \\\n",
       "2016-12-16 03:27:00                29.45   1011.49       6.72        186.0   \n",
       "2016-12-16 03:28:00                29.45   1011.49       6.72        186.0   \n",
       "2016-12-16 03:29:00                29.45   1011.49       6.72        186.0   \n",
       "\n",
       "                     precipIntensity  dewPoint  precipProbability  \n",
       "2016-12-16 03:27:00           0.0101     31.27               0.51  \n",
       "2016-12-16 03:28:00           0.0101     31.27               0.51  \n",
       "2016-12-16 03:29:00           0.0101     31.27               0.51  \n",
       "\n",
       "[3 rows x 28 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "home_dat = home_dat[0:-1] ## == dataset[0:dataset.shape[0]-1] == dataset[0:len(dataset)-1] == dataset[:-1]\n",
    "home_dat.tail(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['use', 'gen', 'House overall', 'Dishwasher', 'Furnace 1', 'Furnace 2',\n",
       "       'Home office', 'Fridge', 'Wine cellar', 'Garage door', 'Kitchen 12',\n",
       "       'Kitchen 14', 'Kitchen 38', 'Barn', 'Well', 'Microwave', 'Living room',\n",
       "       'Solar', 'temperature', 'humidity', 'visibility', 'apparentTemperature',\n",
       "       'pressure', 'windSpeed', 'windBearing', 'precipIntensity', 'dewPoint',\n",
       "       'precipProbability', 'Furnace', 'Kitchen'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "home_dat['Furnace'] = home_dat[['Furnace 1','Furnace 2']].sum(axis=1)\n",
    "home_dat['Kitchen'] = home_dat[['Kitchen 12','Kitchen 14','Kitchen 38']].sum(axis=1)\n",
    "\n",
    "#df['Result'] = df['Column A']/df['Column B']\n",
    "home_dat.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>use</th>\n",
       "      <th>gen</th>\n",
       "      <th>House overall</th>\n",
       "      <th>Dishwasher</th>\n",
       "      <th>Furnace 1</th>\n",
       "      <th>Furnace 2</th>\n",
       "      <th>Home office</th>\n",
       "      <th>Fridge</th>\n",
       "      <th>Wine cellar</th>\n",
       "      <th>Garage door</th>\n",
       "      <th>...</th>\n",
       "      <th>visibility</th>\n",
       "      <th>apparentTemperature</th>\n",
       "      <th>pressure</th>\n",
       "      <th>windSpeed</th>\n",
       "      <th>windBearing</th>\n",
       "      <th>precipIntensity</th>\n",
       "      <th>dewPoint</th>\n",
       "      <th>precipProbability</th>\n",
       "      <th>Furnace</th>\n",
       "      <th>Kitchen</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "      <td>503910.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.858962</td>\n",
       "      <td>0.076229</td>\n",
       "      <td>0.858962</td>\n",
       "      <td>0.031368</td>\n",
       "      <td>0.099210</td>\n",
       "      <td>0.136779</td>\n",
       "      <td>0.081287</td>\n",
       "      <td>0.063556</td>\n",
       "      <td>0.042137</td>\n",
       "      <td>0.014139</td>\n",
       "      <td>...</td>\n",
       "      <td>9.253444</td>\n",
       "      <td>48.263382</td>\n",
       "      <td>1016.301625</td>\n",
       "      <td>6.649936</td>\n",
       "      <td>202.356843</td>\n",
       "      <td>0.002598</td>\n",
       "      <td>38.694013</td>\n",
       "      <td>0.056453</td>\n",
       "      <td>0.235989</td>\n",
       "      <td>0.009788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.058207</td>\n",
       "      <td>0.128428</td>\n",
       "      <td>1.058207</td>\n",
       "      <td>0.190951</td>\n",
       "      <td>0.169059</td>\n",
       "      <td>0.178631</td>\n",
       "      <td>0.104466</td>\n",
       "      <td>0.076199</td>\n",
       "      <td>0.057967</td>\n",
       "      <td>0.014292</td>\n",
       "      <td>...</td>\n",
       "      <td>1.611186</td>\n",
       "      <td>22.027916</td>\n",
       "      <td>7.895185</td>\n",
       "      <td>3.982716</td>\n",
       "      <td>106.520474</td>\n",
       "      <td>0.011257</td>\n",
       "      <td>19.087939</td>\n",
       "      <td>0.165836</td>\n",
       "      <td>0.273885</td>\n",
       "      <td>0.079883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.270000</td>\n",
       "      <td>-32.080000</td>\n",
       "      <td>986.400000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-27.240000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.367667</td>\n",
       "      <td>0.003367</td>\n",
       "      <td>0.367667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.020233</td>\n",
       "      <td>0.064400</td>\n",
       "      <td>0.040383</td>\n",
       "      <td>0.005083</td>\n",
       "      <td>0.007133</td>\n",
       "      <td>0.012733</td>\n",
       "      <td>...</td>\n",
       "      <td>9.420000</td>\n",
       "      <td>31.090000</td>\n",
       "      <td>1011.290000</td>\n",
       "      <td>3.660000</td>\n",
       "      <td>148.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.600000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.085300</td>\n",
       "      <td>0.000600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 use            gen  House overall     Dishwasher  \\\n",
       "count  503910.000000  503910.000000  503910.000000  503910.000000   \n",
       "mean        0.858962       0.076229       0.858962       0.031368   \n",
       "std         1.058207       0.128428       1.058207       0.190951   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%         0.367667       0.003367       0.367667       0.000000   \n",
       "\n",
       "           Furnace 1      Furnace 2    Home office         Fridge  \\\n",
       "count  503910.000000  503910.000000  503910.000000  503910.000000   \n",
       "mean        0.099210       0.136779       0.081287       0.063556   \n",
       "std         0.169059       0.178631       0.104466       0.076199   \n",
       "min         0.000017       0.000067       0.000083       0.000067   \n",
       "25%         0.020233       0.064400       0.040383       0.005083   \n",
       "\n",
       "         Wine cellar    Garage door  ...     visibility  apparentTemperature  \\\n",
       "count  503910.000000  503910.000000  ...  503910.000000        503910.000000   \n",
       "mean        0.042137       0.014139  ...       9.253444            48.263382   \n",
       "std         0.057967       0.014292  ...       1.611186            22.027916   \n",
       "min         0.000017       0.000017  ...       0.270000           -32.080000   \n",
       "25%         0.007133       0.012733  ...       9.420000            31.090000   \n",
       "\n",
       "            pressure      windSpeed    windBearing  precipIntensity  \\\n",
       "count  503910.000000  503910.000000  503910.000000    503910.000000   \n",
       "mean     1016.301625       6.649936     202.356843         0.002598   \n",
       "std         7.895185       3.982716     106.520474         0.011257   \n",
       "min       986.400000       0.000000       0.000000         0.000000   \n",
       "25%      1011.290000       3.660000     148.000000         0.000000   \n",
       "\n",
       "            dewPoint  precipProbability        Furnace        Kitchen  \n",
       "count  503910.000000      503910.000000  503910.000000  503910.000000  \n",
       "mean       38.694013           0.056453       0.235989       0.009788  \n",
       "std        19.087939           0.165836       0.273885       0.079883  \n",
       "min       -27.240000           0.000000       0.000300       0.000000  \n",
       "25%        24.600000           0.000000       0.085300       0.000600  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DescriptiveStats=home_dat.describe()\n",
    "DescriptiveStats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "CleanData_per_min = home_dat.filter(items=[ 'gen', 'use', 'House overall', \n",
    "                               'Furnace', 'Home office',  'Living room',\n",
    "                               'Wine cellar', 'Garage door', 'Kitchen',\n",
    "                               'Barn', 'Well', 'Fridge', 'Microwave', 'Dishwasher',\n",
    "                               'temperature', 'humidity', \n",
    "                               'visibility', 'apparentTemperature',\n",
    "                               'pressure', 'windSpeed', 'windBearing', 'precipIntensity', 'dewPoint'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "WeatherData_per_min= home_dat.filter(items=[ 'gen', 'temperature', 'humidity', \n",
    "                               'visibility', 'apparentTemperature',\n",
    "                               'pressure', 'windSpeed', 'windBearing', 'precipIntensity', 'dewPoint'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "RoomsData_per_min = home_dat.filter(items=[ 'gen', 'House overall', \n",
    "                               'Furnace', 'Home office',  'Living room',\n",
    "                               'Wine cellar', 'Garage door', 'Kitchen',\n",
    "                               'Barn', 'Well', 'Fridge', 'Microwave', 'Dishwasher'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_dat['EneEffi'] = home_dat['use']-home_dat['gen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "EnergyData_per_min = home_dat.filter(items=['EneEffi', 'gen', 'use', 'House overall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EneEffi</th>\n",
       "      <th>gen</th>\n",
       "      <th>use</th>\n",
       "      <th>House overall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:27:00</th>\n",
       "      <td>1.921050</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.924267</td>\n",
       "      <td>1.924267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:28:00</th>\n",
       "      <td>1.974983</td>\n",
       "      <td>0.003217</td>\n",
       "      <td>1.978200</td>\n",
       "      <td>1.978200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-12-16 03:29:00</th>\n",
       "      <td>1.987717</td>\n",
       "      <td>0.003233</td>\n",
       "      <td>1.990950</td>\n",
       "      <td>1.990950</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      EneEffi       gen       use  House overall\n",
       "2016-12-16 03:27:00  1.921050  0.003217  1.924267       1.924267\n",
       "2016-12-16 03:28:00  1.974983  0.003217  1.978200       1.978200\n",
       "2016-12-16 03:29:00  1.987717  0.003233  1.990950       1.990950"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EnergyData.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summaries\n",
    "# Generate Data per (day) and (month) \n",
    "CleanData_per_day = CleanData.resample('D').sum()\n",
    "WeatherData_per_day = WeatherData.resample('D').sum()\n",
    "RoomsData_per_day = WeatherData.resample('D').sum()\n",
    "EnergyData_per_day = WeatherData.resample('D').sum()\n",
    "\n",
    "CleanData_per_month = CleanData.resample('M').sum()\n",
    "WeatherData_per_month = WeatherData.resample('M').sum()\n",
    "RoomsData_per_month = WeatherData.resample('M').sum()\n",
    "EnergyData_per_month = WeatherData.resample('M').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save Data to Directory\n",
    "CleanData_per_min.to_csv  (r'CleanData_per_min.csv')\n",
    "EnergyData_per_min.to_csv (r'EnergyData_per_min.csv')\n",
    "RoomsData_per_min.to_csv  (r'RoomsData_per_min.csv')\n",
    "WeatherData_per_min.to_csv(r'WeatherData_per_min.csv')\n",
    "\n",
    "CleanData_per_day.to_csv  (r'CleanData_per_day.csv')\n",
    "EnergyData_per_day.to_csv (r'EnergyData_per_day.csv')\n",
    "RoomsData_per_day.to_csv  (r'RoomsData_per_day.csv')\n",
    "WeatherData_per_day.to_csv(r'WeatherData_per_day.csv')\n",
    "\n",
    "CleanData_per_month.to_csv  (r'CleanData_per_month.csv')\n",
    "EnergyData_per_month.to_csv (r'EnergyData_per_month.csv')\n",
    "RoomsData_per_month.to_csv  (r'RoomsData_per_month.csv')\n",
    "WeatherData_per_month.to_csv(r'WeatherData_per_month.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gen                    float64\n",
       "use                    float64\n",
       "House overall          float64\n",
       "Furnace                float64\n",
       "Home office            float64\n",
       "Living room            float64\n",
       "Wine cellar            float64\n",
       "Garage door            float64\n",
       "Kitchen                float64\n",
       "Barn                   float64\n",
       "Well                   float64\n",
       "Fridge                 float64\n",
       "Microwave              float64\n",
       "Dishwasher             float64\n",
       "temperature            float64\n",
       "humidity               float64\n",
       "visibility             float64\n",
       "apparentTemperature    float64\n",
       "pressure               float64\n",
       "windSpeed              float64\n",
       "windBearing            float64\n",
       "precipIntensity        float64\n",
       "dewPoint               float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data Preparation\n",
    "CleanData_per_month.columns\n",
    "CleanData_per_month.dtypes \n",
    "#CleanData.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running RF with all the data to investigate the driving variables of energy efficiency at a 1min resolution\n",
    "#my_data.dtypes \n",
    "features=CleanData_per_month\n",
    "type(features)\n",
    "#features.iloc[:,5:].head(5)\n",
    "#features.head(5)\n",
    "#features.info()\n",
    "features.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Labels are the values we want to predict\n",
    "labels = np.array(features['gen'])\n",
    "# Remove the labels from the features\n",
    "# axis 1 refers to the columns\n",
    "features= features.drop('gen', axis = 1)\n",
    "# Saving feature names for later use\n",
    "feature_list = list(features.columns)\n",
    "# Convert to numpy array\n",
    "features = np.array(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training and Testing Sets\n",
    "# Using Skicit-learn to split data into training and testing sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Split the data into training and testing sets\n",
    "train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 0.25, random_state = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Training Features Shape:', train_features.shape)\n",
    "print('Training Labels Shape:', train_labels.shape)\n",
    "print('Testing Features Shape:', test_features.shape)\n",
    "print('Testing Labels Shape:', test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the model we are using\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "# Instantiate model with 10 decision trees\n",
    "#rf = RandomForestRegressor(n_estimators = 100, random_state = 100)\n",
    "rf = RandomForestRegressor(n_estimators = 100, random_state = 100)\n",
    "# Train the model on training data\n",
    "rf.fit(train_features, train_labels);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the forest's predict method on the test data\n",
    "predictions = rf.predict(test_features)\n",
    "# Calculate the absolute errors\n",
    "errors = abs(predictions - test_labels)\n",
    "# Print out the mean absolute error (mae)\n",
    "print('Mean Absolute Error:', round(np.mean(errors), 2), 'degrees.')\n",
    "#Mean Absolute Error: 8512.05 degrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate mean absolute percentage error (MAPE)\n",
    "mape = 100 * (errors / test_labels)\n",
    "# Calculate and display accuracy\n",
    "accuracy = 100 - np.mean(mape)\n",
    "print('Accuracy:', round(accuracy, 2), '%.')\n",
    "#Accuracy: 95.96 %.\n",
    "    # That looks pretty good! Our model has learned how to predict the List price as a function of list and close price in Houston with 95.96% accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot = all_energy_effi.plot(kind = \"bar\", figsize = (5,5))\n",
    "plot.set_title(\"energy efficiency\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What are the variables that regulate power generation?\n",
    "# Import tools needed for visualization\n",
    "from sklearn.tree import export_graphviz\n",
    "import pydot\n",
    "# Pull out one tree from the forest\n",
    "tree = rf.estimators_[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import tools needed for visualization\n",
    "from sklearn.tree import export_graphviz\n",
    "import pydot\n",
    "# Pull out one tree from the forest\n",
    "tree = rf.estimators_[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export the image to a dot file\n",
    "export_graphviz(tree, out_file = 'tree', feature_names = feature_list, rounded = True, precision = 1)\n",
    "# Use dot file to create a graph\n",
    "(graph, ) = pydot.graph_from_dot_file('tree')\n",
    "# Write graph to a png file\n",
    "graph.write_png('tree.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get numerical feature importances\n",
    "importances = list(rf.feature_importances_)\n",
    "# List of tuples with variable and importance\n",
    "feature_importances = [(feature, round(importance, 2)) for feature, importance in zip(feature_list, importances)]\n",
    "# Sort the feature importances by most important first\n",
    "feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)\n",
    "# Print out the feature and importances \n",
    "[print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importances];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure();\n",
    "feature_importances.plot();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# New random forest with only the two most important variables\n",
    "rf_most_important = RandomForestRegressor(n_estimators= 1000, random_state=1000)\n",
    "# Extract the two most important features\n",
    "important_indices = [feature_list.index('temperature'), feature_list.index('dewPoint')]\n",
    "train_important = train_features[:, important_indices]\n",
    "test_important = test_features[:, important_indices]\n",
    "# Train the random forest\n",
    "rf_most_important.fit(train_important, train_labels)\n",
    "# Make predictions and determine the error\n",
    "predictions = rf_most_important.predict(test_important)\n",
    "errors = abs(predictions - test_labels)\n",
    "# Display the performance metrics\n",
    "print('Mean Absolute Error:', round(np.mean(errors), 2), 'degrees.')\n",
    "mape = np.mean(100 * (errors / test_labels))\n",
    "accuracy = 100 - mape\n",
    "print('Accuracy:', round(accuracy, 2), '%.')\n",
    "\n",
    "#Note:\n",
    "#This tells us that temperature and dew point are the two most important variables to regulate generated energy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model\n",
    "#model.save(\"EnergyGenerated_trained.h5\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}