Logistic-Regression-with-neural-network(Foto-detection).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear regression ML "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py\n",
    "import scipy\n",
    "from PIL import Image\n",
    "from scipy import ndimage\n",
    "from lr_utils import load_dataset\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data and inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the data (cat/non-cat)\n",
    "train_set_x_orig, train_set_y, test_set_x_orig, test_set_y, classes = load_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of a picture\n",
    "index = 25\n",
    "plt.imshow(train_set_x_orig[index])\n",
    "print (\"y = \" + str(train_set_y[:, index]) + \", it's a '\" + classes[np.squeeze(train_set_y[:, index])].decode(\"utf-8\") +  \"' picture.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_train = train_set_y.shape[1]\n",
    "m_test = test_set_y.shape[1]\n",
    "num_px = train_set_x_orig.shape[1]\n",
    "\n",
    "print (\"Number of training examples: m_train = \" + str(m_train))\n",
    "print (\"Number of testing examples: m_test = \" + str(m_test))\n",
    "print (\"Height/Width of each image: num_px = \" + str(num_px))\n",
    "print (\"Each image is of size: (\" + str(num_px) + \", \" + str(num_px) + \", 3)\")\n",
    "print (\"train_set_x shape: \" + str(train_set_x_orig.shape))\n",
    "print (\"train_set_y shape: \" + str(train_set_y.shape))\n",
    "print (\"test_set_x shape: \" + str(test_set_x_orig.shape))\n",
    "print (\"test_set_y shape: \" + str(test_set_y.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reshaping / wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape the training and test examples\n",
    "\n",
    "train_set_x_flatten = train_set_x_orig.reshape(train_set_x_orig.shape[0], -1 ).T\n",
    "test_set_x_flatten = test_set_x_orig.reshape(test_set_x_orig.shape[0], -1).T\n",
    "\n",
    "print (\"train_set_x_flatten shape: \" + str(train_set_x_flatten.shape))\n",
    "print (\"train_set_y shape: \" + str(train_set_y.shape))\n",
    "print (\"test_set_x_flatten shape: \" + str(test_set_x_flatten.shape))\n",
    "print (\"test_set_y shape: \" + str(test_set_y.shape))\n",
    "print (\"sanity check after reshaping: \" + str(train_set_x_flatten[0:5,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardize our dataset. Just for pictures else stanardize and center dataset\n",
    "# substract the mean of the whole numpy array from each example, and then divide each example by the standard deviation of the whole numpy array\n",
    "train_set_x = train_set_x_flatten/255.\n",
    "\n",
    "test_set_x = test_set_x_flatten/255."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sigmoid function (activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    \"\"\"\n",
    "    Compute the sigmoid of z\n",
    "\n",
    "    Arguments:\n",
    "    z -- A scalar or numpy array of any size.\n",
    "\n",
    "    Return:\n",
    "    s -- sigmoid(z)\n",
    "    \"\"\"\n",
    "\n",
    "    s = 1 / (1 + np.exp(-z))\n",
    "    \n",
    "    return s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_with_zeros(dim):\n",
    "    \"\"\"\n",
    "    This function creates a vector of zeros of shape (dim, 1) for w and initializes b to 0.\n",
    "    \n",
    "    Argument:\n",
    "    dim -- size of the w vector we want (or number of parameters in this case)\n",
    "    \n",
    "    Returns:\n",
    "    w -- initialized vector of shape (dim, 1)\n",
    "    b -- initialized scalar (corresponds to the bias)\n",
    "    \"\"\"\n",
    "\n",
    "    w = np.zeros((dim,1))\n",
    "    b = 0\n",
    "\n",
    "    assert(w.shape == (dim, 1))\n",
    "    assert(isinstance(b, float) or isinstance(b, int))\n",
    "    \n",
    "    return w, b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Forward and Backward propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def propagate(w, b, X, Y):\n",
    "    \"\"\"\n",
    "    Implement the cost function and its gradient for the propagation explained above\n",
    "\n",
    "    Arguments:\n",
    "    w -- weights, a numpy array of size (num_px * num_px * 3, 1)\n",
    "    b -- bias, a scalar\n",
    "    X -- data of size (num_px * num_px * 3, number of examples)\n",
    "    Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat) of size (1, number of examples)\n",
    "\n",
    "    Return:\n",
    "    cost -- negative log-likelihood cost for logistic regression\n",
    "    dw -- gradient of the loss with respect to w, thus same shape as w\n",
    "    db -- gradient of the loss with respect to b, thus same shape as b\n",
    "    \n",
    "    Tips:\n",
    "    - Write your code step by step for the propagation. np.log(), np.dot()\n",
    "    \"\"\"\n",
    "    \n",
    "    m = X.shape[1]\n",
    "    \n",
    "    # FORWARD PROPAGATION (FROM X TO COST)\n",
    "    A = sigmoid(np.dot(w.T, X) + b)                  # compute activation\n",
    "    cost = (-1/m) * np.sum(Y * np.log(A) + (1-Y) * (np.log(1-A))) # compute cost\n",
    "    \n",
    "    # BACKWARD PROPAGATION (TO FIND GRAD)\n",
    "    dw = (1/m) * np.dot(X , (A - Y).T)\n",
    "    db = (1/m) * np.sum(A - Y)\n",
    "\n",
    "    assert(dw.shape == w.shape)\n",
    "    assert(db.dtype == float)\n",
    "    cost = np.squeeze(cost)\n",
    "    assert(cost.shape == ())\n",
    "    \n",
    "    grads = {\"dw\": dw,\n",
    "             \"db\": db}\n",
    "    \n",
    "    return grads, cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize(w, b, X, Y, num_iterations, learning_rate, print_cost = False):\n",
    "    \"\"\"\n",
    "    This function optimizes w and b by running a gradient descent algorithm\n",
    "    \n",
    "    Arguments:\n",
    "    w -- weights, a numpy array of size (num_px * num_px * 3, 1)\n",
    "    b -- bias, a scalar\n",
    "    X -- data of shape (num_px * num_px * 3, number of examples)\n",
    "    Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat), of shape (1, number of examples)\n",
    "    num_iterations -- number of iterations of the optimization loop\n",
    "    learning_rate -- learning rate of the gradient descent update rule\n",
    "    print_cost -- True to print the loss every 100 steps\n",
    "    \n",
    "    Returns:\n",
    "    params -- dictionary containing the weights w and bias b\n",
    "    grads -- dictionary containing the gradients of the weights and bias with respect to the cost function\n",
    "    costs -- list of all the costs computed during the optimization, this will be used to plot the learning curve.\n",
    "    \n",
    "    Tips:\n",
    "    You basically need to write down two steps and iterate through them:\n",
    "        1) Calculate the cost and the gradient for the current parameters. Use propagate().\n",
    "        2) Update the parameters using gradient descent rule for w and b.\n",
    "    \"\"\"\n",
    "    \n",
    "    costs = []\n",
    "    \n",
    "    for i in range(num_iterations):\n",
    "        \n",
    "        \n",
    "        # Cost and gradient calculation \n",
    "        grads, cost = propagate(w, b, X, Y)\n",
    "        #print(cost,costs)\n",
    "        \n",
    "        # Retrieve derivatives from grads\n",
    "        dw = grads[\"dw\"]\n",
    "        db = grads[\"db\"]\n",
    "        \n",
    "        # update rule \n",
    "        w = w - (learning_rate * dw)\n",
    "        b = b - (learning_rate * db)\n",
    "        \n",
    "        # Record the costs\n",
    "        if i % 100 == 0:\n",
    "            costs.append(cost)\n",
    "        \n",
    "        # Print the cost every 100 training iterations\n",
    "        if print_cost and i % 100 == 0:\n",
    "            print (\"Cost after iteration %i: %f\" %(i, cost))\n",
    "    \n",
    "    params = {\"w\": w,\n",
    "              \"b\": b}\n",
    "    \n",
    "    grads = {\"dw\": dw,\n",
    "             \"db\": db}\n",
    "    \n",
    "    return params, grads, costs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(w, b, X):\n",
    "    '''\n",
    "    Predict whether the label is 0 or 1 using learned logistic regression parameters (w, b)\n",
    "    \n",
    "    Arguments:\n",
    "    w -- weights, a numpy array of size (num_px * num_px * 3, 1)\n",
    "    b -- bias, a scalar\n",
    "    X -- data of size (num_px * num_px * 3, number of examples)\n",
    "    \n",
    "    Returns:\n",
    "    Y_prediction -- a numpy array (vector) containing all predictions (0/1) for the examples in X\n",
    "    '''\n",
    "    \n",
    "    m = X.shape[1]\n",
    "    Y_prediction = np.zeros((1,m))\n",
    "    w = w.reshape(X.shape[0], -1)\n",
    "    \n",
    "    # Compute vector \"A\" predicting the probabilities of a cat being present in the picture\n",
    "    A = sigmoid(np.dot(w.T ,X) +b)\n",
    "\n",
    "    \n",
    "    for i in range(A.shape[1]):\n",
    "        \n",
    "        # Convert probabilities A[0,i] to actual predictions p[0,i]\n",
    "        if A[0,i] <= 0.5:\n",
    "            Y_prediction[0,i] = 0\n",
    "        else:\n",
    "            Y_prediction[0,i] = 1\n",
    "    \n",
    "    assert(Y_prediction.shape == (1, m))\n",
    "    \n",
    "    return Y_prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model to call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(X_train, Y_train, X_test, Y_test, num_iterations = 2000, learning_rate = 0.5, print_cost = False):\n",
    "    \"\"\"\n",
    "    Builds the logistic regression model by calling the function you've implemented previously\n",
    "    \n",
    "    Arguments:\n",
    "    X_train -- training set represented by a numpy array of shape (num_px * num_px * 3, m_train)\n",
    "    Y_train -- training labels represented by a numpy array (vector) of shape (1, m_train)\n",
    "    X_test -- test set represented by a numpy array of shape (num_px * num_px * 3, m_test)\n",
    "    Y_test -- test labels represented by a numpy array (vector) of shape (1, m_test)\n",
    "    num_iterations -- hyperparameter representing the number of iterations to optimize the parameters\n",
    "    learning_rate -- hyperparameter representing the learning rate used in the update rule of optimize()\n",
    "    print_cost -- Set to true to print the cost every 100 iterations\n",
    "    \n",
    "    Returns:\n",
    "    d -- dictionary containing information about the model.\n",
    "    \"\"\"\n",
    "     \n",
    "    # initialize parameters with zeros \n",
    "    w, b = initialize_with_zeros(X_train.shape[0])\n",
    "    \n",
    "    # Gradient descent \n",
    "    parameters, grads, costs = optimize(w, b, X_train, Y_train, num_iterations, learning_rate, print_cost)\n",
    "    \n",
    "    # Retrieve parameters w and b from dictionary \"parameters\"\n",
    "    w = parameters[\"w\"]\n",
    "    b = parameters[\"b\"]\n",
    "    \n",
    "    # Predict test/train set examples (≈ 2 lines of code)\n",
    "    Y_prediction_test = predict(w, b, X_test)\n",
    "    Y_prediction_train = predict(w, b, X_train)\n",
    "\n",
    "    # Print train/test Errors\n",
    "    print(\"train accuracy: {} %\".format(100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100))\n",
    "    print(\"test accuracy: {} %\".format(100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100))\n",
    "\n",
    "    \n",
    "    d = {\"costs\": costs,\n",
    "         \"Y_prediction_test\": Y_prediction_test, \n",
    "         \"Y_prediction_train\" : Y_prediction_train, \n",
    "         \"w\" : w, \n",
    "         \"b\" : b,\n",
    "         \"learning_rate\" : learning_rate,\n",
    "         \"num_iterations\": num_iterations}\n",
    "    \n",
    "    return d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = 2000, learning_rate = 0.005, print_cost = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of a picture \n",
    "index = 1\n",
    "plt.imshow(test_set_x[:,index].reshape((num_px, num_px, 3)))\n",
    "print (\"y = \" + str(test_set_y[0,index]) + \", you predicted that it is a \\\"\" + classes[d[\"Y_prediction_test\"][0,index]].decode(\"utf-8\") +  \"\\\" picture.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## learning Rate (cost function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot learning curve (with costs)\n",
    "costs = np.squeeze(d['costs'])\n",
    "plt.plot(costs)\n",
    "plt.ylabel('cost')\n",
    "plt.xlabel('iterations (per hundreds)')\n",
    "plt.title(\"Learning rate =\" + str(d[\"learning_rate\"]))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Choice of learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rates = [0.01, 0.001, 0.0001] #should be expanded for small models\n",
    "models = {}\n",
    "for i in learning_rates:\n",
    "    print (\"learning rate is: \" + str(i))\n",
    "    models[str(i)] = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = 1500, learning_rate = i, print_cost = False)\n",
    "    print ('\\n' + \"-------------------------------------------------------\" + '\\n')\n",
    "\n",
    "for i in learning_rates:\n",
    "    plt.plot(np.squeeze(models[str(i)][\"costs\"]), label= str(models[str(i)][\"learning_rate\"]))\n",
    "\n",
    "plt.ylabel('cost')\n",
    "plt.xlabel('iterations (hundreds)')\n",
    "\n",
    "legend = plt.legend(loc='upper center', shadow=True)\n",
    "frame = legend.get_frame()\n",
    "frame.set_facecolor('0.90')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### test a new picture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_image = \"my_image.jpg\"   # change this to the name of your image file \n",
    "\n",
    "# We preprocess the image to fit your algorithm.\n",
    "fname = \"images/\" + my_image\n",
    "image = np.array(ndimage.imread(fname, flatten=False))\n",
    "my_image = scipy.misc.imresize(image, size=(num_px,num_px)).reshape((1, num_px*num_px*3)).T\n",
    "my_predicted_image = predict(d[\"w\"], d[\"b\"], my_image)\n",
    "\n",
    "plt.imshow(image)\n",
    "print(\"y = \" + str(np.squeeze(my_predicted_image)) + \", your algorithm predicts a \\\"\" + classes[int(np.squeeze(my_predicted_image)),].decode(\"utf-8\") +  \"\\\" picture.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}