Exp_18_Singular_Value_Decomposition_SVD_Ex.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "552497fb",
   "metadata": {},
   "source": [
    "# Singular Value Decomposition -> SVD from Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c317a5e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.linalg import svd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3da4b9df",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "006e5ded",
   "metadata": {},
   "outputs": [],
   "source": [
    "A = np.array([\n",
    "    [1,2],\n",
    "    [3,4],\n",
    "    [5,6]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f28396aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 2],\n",
       "       [3, 4],\n",
       "       [5, 6]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "768e4db3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.2298477 ,  0.88346102,  0.40824829],\n",
       "       [-0.52474482,  0.24078249, -0.81649658],\n",
       "       [-0.81964194, -0.40189603,  0.40824829]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "U, Sigma, VT = svd(A)\n",
    "U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6da4e245",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.52551809, 0.51430058])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83b5e1b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.61962948, -0.78489445],\n",
       "       [-0.78489445,  0.61962948]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3aa64e2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0.],\n",
       "       [0., 0.],\n",
       "       [0., 0.]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigma = np.zeros((A.shape[0],A.shape[1]))\n",
    "sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ac2523c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.52551809, 0.        ],\n",
       "       [0.        , 0.51430058],\n",
       "       [0.        , 0.        ]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigma[:A.shape[1],:A.shape[1]] = np.diag(Sigma)\n",
    "sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f32a02b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2.],\n",
       "       [3., 4.],\n",
       "       [5., 6.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from numpy import dot\n",
    "A_rec = U.dot(sigma.dot(VT))\n",
    "A_rec"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ee5de8",
   "metadata": {},
   "source": [
    "# SVD for Dimensionality Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e0215b88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  2  3  4  5  6  7  8  9 10]\n",
      " [11 12 13 14 15 16 17 18 19 20]\n",
      " [21 22 23 24 25 26 27 28 29 30]]\n"
     ]
    }
   ],
   "source": [
    "# define a matrix\n",
    "A = np.array([\n",
    "    [1,2,3,4,5,6,7,8,9,10],\n",
    "    [11,12,13,14,15,16,17,18,19,20],\n",
    "    [21,22,23,24,25,26,27,28,29,30]\n",
    "])\n",
    "print(A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "38708a3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.19101157, -0.89266338,  0.40824829],\n",
       "       [-0.51371859, -0.26348917, -0.81649658],\n",
       "       [-0.8364256 ,  0.36568503,  0.40824829]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "U, Sigma, VT = svd(A)\n",
    "U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2fe923a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.69657342e+01, 7.25578339e+00, 1.48879510e-15])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "af04ecfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.24139304, -0.25728686, -0.27318068, -0.2890745 , -0.30496832,\n",
       "        -0.32086214, -0.33675595, -0.35264977, -0.36854359, -0.38443741],\n",
       "       [ 0.53589546,  0.42695236,  0.31800926,  0.20906617,  0.10012307,\n",
       "        -0.00882003, -0.11776313, -0.22670623, -0.33564933, -0.44459242],\n",
       "       [ 0.09975293, -0.01037753,  0.23987452,  0.05273845,  0.0996209 ,\n",
       "        -0.46517509, -0.52300449, -0.23591804,  0.593137  ,  0.14935136],\n",
       "       [-0.259848  , -0.19489762, -0.10988774,  0.9273013 , -0.07153728,\n",
       "        -0.05251768, -0.04829915, -0.05415057, -0.07582496, -0.0603383 ],\n",
       "       [-0.26312174, -0.0585767 , -0.12150761, -0.05740164,  0.9302533 ,\n",
       "        -0.0441965 , -0.05005435, -0.07728074, -0.13808364, -0.12003039],\n",
       "       [-0.34412869, -0.09157816,  0.43274424, -0.05958094, -0.05492318,\n",
       "         0.70339002, -0.33412339, -0.23272698,  0.08693936, -0.10601229],\n",
       "       [-0.3607094 ,  0.01575684,  0.51799467, -0.04727562, -0.05090152,\n",
       "        -0.33292338,  0.61579275, -0.27850803,  0.07385978, -0.15308609],\n",
       "       [-0.33345757,  0.21857007,  0.28415895, -0.02511566, -0.05422893,\n",
       "        -0.22221261, -0.27509879,  0.75032187, -0.10121342, -0.24172391],\n",
       "       [-0.23733137,  0.57140887, -0.45105898,  0.01252895, -0.06910398,\n",
       "         0.11951822,  0.08414965, -0.10361154,  0.46917172, -0.39567153],\n",
       "       [-0.30296011,  0.57190501, -0.00875539,  0.01380706, -0.05685882,\n",
       "        -0.08131326, -0.14406831, -0.23288126, -0.36263916,  0.60376424]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "26f006eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create m x n Sigma matrix\n",
    "sigma = np.zeros((A.shape[0], A.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "557dd3cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.69657342e+01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        0.00000000e+00, 0.00000000e+00],\n",
       "       [0.00000000e+00, 7.25578339e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        0.00000000e+00, 0.00000000e+00],\n",
       "       [0.00000000e+00, 0.00000000e+00, 1.48879510e-15, 0.00000000e+00,\n",
       "        0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        0.00000000e+00, 0.00000000e+00]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# populate Sigma with n x n diagonal matrix\n",
    "sigma[:A.shape[0], :A.shape[0]] = np.diag(Sigma)\n",
    "sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8d8b2d4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.24139304, -0.25728686, -0.27318068, -0.2890745 , -0.30496832,\n",
       "        -0.32086214, -0.33675595, -0.35264977, -0.36854359, -0.38443741],\n",
       "       [ 0.53589546,  0.42695236,  0.31800926,  0.20906617,  0.10012307,\n",
       "        -0.00882003, -0.11776313, -0.22670623, -0.33564933, -0.44459242]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select\n",
    "n_elements = 2\n",
    "sigma = sigma[:, :n_elements]\n",
    "VT = VT[:n_elements, :]\n",
    "VT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "71c86fc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[96.96573419,  0.        ],\n",
       "       [ 0.        ,  7.25578339],\n",
       "       [ 0.        ,  0.        ]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reconstruct\n",
    "B = U.dot(sigma.dot(VT))\n",
    "sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6846b9c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.69657342e+01, 7.25578339e+00, 1.48879510e-15])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "14ccc89e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10.],\n",
       "       [11., 12., 13., 14., 15., 16., 17., 18., 19., 20.],\n",
       "       [21., 22., 23., 24., 25., 26., 27., 28., 29., 30.]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c8922e10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10],\n",
       "       [11, 12, 13, 14, 15, 16, 17, 18, 19, 20],\n",
       "       [21, 22, 23, 24, 25, 26, 27, 28, 29, 30]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "75182cf2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-24.99854961 -51.72492049 -78.45129137]\n"
     ]
    }
   ],
   "source": [
    "# transform\n",
    "T = U.dot(Sigma)\n",
    "print(T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "34943e4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-18.52157747  -6.47697214]\n",
      " [-49.81310011  -1.91182038]\n",
      " [-81.10462276   2.65333138]]\n"
     ]
    }
   ],
   "source": [
    "T = A.dot(VT.T)\n",
    "print(T)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84056d47",
   "metadata": {},
   "source": [
    "### 1. Two words ‘black’ and ‘brown’ are encoded in integer form as [3 2 2] and [2 3 -2] respectively. Compute the SVD on the combined integer features and report the results.\n",
    "### 2. Compute SVD on two-word vectors represented as integers [-3 6 6].T and [1 -2 -2].T. where ‘.T’ is the python representation of transpose.\n",
    "### 3. Two words in two documents are one – hot encoded as [1 0 1 0] and [0 1 0 1]. Compute the SVD transformation and find the semantic representation of the two words.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79ebdde3",
   "metadata": {},
   "source": [
    "# ALM: Impliment the steps of SVD using python Code and verify the results with the sklearn API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d650211b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}