Reading1M_feature_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import spotipy\n",
    "import spotipy.oauth2 as oauth2\n",
    "from spotipy.oauth2 import SpotifyOAuth,SpotifyClientCredentials\n",
    "import yaml\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import multiprocessing as mp\n",
    "import time\n",
    "import random\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stream= open(\"spotify.yaml\")\n",
    "spotify_details = yaml.safe_load(stream)\n",
    "auth_manager = SpotifyClientCredentials(client_id=spotify_details['Client_id'],\n",
    "                                        client_secret=spotify_details['client_secret'])\n",
    "sp = spotipy.client.Spotify(auth_manager=auth_manager)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Reading the 1 million playlists and keeping only the unique track URIs for the content-based recommendation system.\n",
    "- The first for loop Read the 1,000 JSON files one at a time.\n",
    "- The second for loop is for getting only the unique track."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loop_slices(path, num_slices=20):\n",
    "      cnt = 0\n",
    "      cnt1 = 0\n",
    "      mpd_playlists = []\n",
    "      unique_tracks= pd.DataFrame()\n",
    "      filenames = os.listdir(path)\n",
    "      for fname in tqdm(sorted(filenames, key=len)):\n",
    "        if fname.startswith(\"mpd.slice.\") and fname.endswith(\".json\"):\n",
    "          cnt += 1\n",
    "          fullpath = os.sep.join((path, fname))\n",
    "          f = open(fullpath)\n",
    "          js = f.read()\n",
    "          f.close()\n",
    "          current_slice = json.loads(js)\n",
    "          # Create a list of all playlists\n",
    "          for playlist in current_slice['playlists']:\n",
    "            cnt1 +=1\n",
    "            mpd_playlists.append(playlist)\n",
    "            if cnt1 == 1000:\n",
    "              cnt1=0\n",
    "              temp=pd.DataFrame(mpd_playlists)\n",
    "              temp=temp.explode('tracks')\n",
    "              temp=pd.DataFrame(temp['tracks'].apply(pd.Series))\n",
    "              unique_tracks=pd.concat([unique_tracks,temp],axis=0)\n",
    "              unique_tracks.drop_duplicates(subset=['track_uri'],inplace=True)\n",
    "              mpd_playlists = []\n",
    "          if cnt == num_slices:\n",
    "            break\n",
    "        return unique_tracks\n",
    "# Path where the json files are extracted\n",
    "path = 'C:\\Spotify-Recommendation-System\\Dataset'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'to_csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-aba7c8fc3b4c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloop_slices\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_slices\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Dataset/1m.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'to_csv'"
     ]
    }
   ],
   "source": [
    "df = loop_slices(path, num_slices=1000)\n",
    "df.to_csv('Dataset/1m.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'track_uri'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3802\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3801\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 3802\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   3803\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\_libs\\index.pyx:138\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\_libs\\index.pyx:165\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:5745\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:5753\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'track_uri'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [13], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrack_uri\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mdf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtrack_uri\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: re\u001b[38;5;241m.\u001b[39mfindall(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+$\u001b[39m\u001b[38;5;124m'\u001b[39m, x)[\u001b[38;5;241m0\u001b[39m])\n\u001b[0;32m      2\u001b[0m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124martist_uri\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124martist_uri\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: re\u001b[38;5;241m.\u001b[39mfindall(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+$\u001b[39m\u001b[38;5;124m'\u001b[39m, x)[\u001b[38;5;241m0\u001b[39m])\n\u001b[0;32m      3\u001b[0m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malbum_uri\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malbum_uri\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: re\u001b[38;5;241m.\u001b[39mfindall(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+$\u001b[39m\u001b[38;5;124m'\u001b[39m, x)[\u001b[38;5;241m0\u001b[39m])\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\frame.py:3807\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3805\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\u001b[38;5;241m.\u001b[39mnlevels \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m   3806\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_getitem_multilevel(key)\n\u001b[1;32m-> 3807\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   3808\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_integer(indexer):\n\u001b[0;32m   3809\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m [indexer]\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3804\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3802\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine\u001b[38;5;241m.\u001b[39mget_loc(casted_key)\n\u001b[0;32m   3803\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m-> 3804\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[0;32m   3805\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[0;32m   3806\u001b[0m     \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[0;32m   3807\u001b[0m     \u001b[38;5;66;03m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[0;32m   3808\u001b[0m     \u001b[38;5;66;03m#  the TypeError.\u001b[39;00m\n\u001b[0;32m   3809\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_indexing_error(key)\n",
      "\u001b[1;31mKeyError\u001b[0m: 'track_uri'"
     ]
    }
   ],
   "source": [
    "df[\"track_uri\"] = df[\"track_uri\"].apply(lambda x: re.findall(r'\\w+$', x)[0])\n",
    "df[\"artist_uri\"] = df[\"artist_uri\"].apply(lambda x: re.findall(r'\\w+$', x)[0])\n",
    "df[\"album_uri\"] = df[\"album_uri\"].apply(lambda x: re.findall(r'\\w+$', x)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('Data/1mV3.csv') #Dropped all columns except ['track_uri', 'artist_uri', 'album_uri']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['track_uri', 'artist_uri', 'album_uri'], dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_uri=df[\"track_uri\"].unique()\n",
    "a_uri=df[\"artist_uri\"].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using the Spotify API for Feature Extraction and Saving Results to a CSV File and Errors to a Log File\n",
    "\n",
    "I was using SP.track first, but I realised that it would take a lot of time and I would have to counter a lot of Api rate limits, so I used SP.tracks and SP.artists instead. They accept lists with a 50-URI maximum and handle them in a single request, so it took a lot less time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/audio_features.csv','a')\n",
    "e=0\n",
    "for i in tqdm(range(0,len(t_uri),100)):\n",
    "    try:\n",
    "     track_feature = sp.audio_features(t_uri[i:i+100])\n",
    "     track_df = pd.DataFrame(track_feature)\n",
    "     csv_data = track_df.to_csv(header=False,index=False)\n",
    "     f.write(csv_data)\n",
    "    except Exception as error:\n",
    "        e+=1\n",
    "        r = open(\"audio_features_log.txt\", \"a\")\n",
    "        r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\": \"+str(error)+'\\n')\n",
    "        r.close()\n",
    "        time.sleep(3)\n",
    "        continue\n",
    "r = open(\"audio_features_log.txt\", \"a\")\n",
    "r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\" _________________________ \"+\"Total Number Of Errors : \"+str(e)+\" _________________________ \"+'\\n')\n",
    "r.close()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/track_features.csv','a')\n",
    "e=0\n",
    "for i in tqdm(range(0,len(t_uri),50)):\n",
    "    try:\n",
    "        track_features = sp.tracks(t_uri[i:i+50])\n",
    "        for x in range(50):\n",
    "            track_pop=pd.DataFrame([t_uri[i+x]])\n",
    "            track_pop['release_date']=track_features['tracks'][x]['album']['release_date']\n",
    "            track_pop['pop'] = track_features['tracks'][x][\"popularity\"]\n",
    "            csv_data = track_pop.to_csv(header=False,index=False)\n",
    "            f.write(csv_data)\n",
    "    except Exception as error:\n",
    "        e+=1\n",
    "        r = open(\"track_features.txt\", \"a\")\n",
    "        r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\": \"+str(error)+'\\n')\n",
    "        r.close()\n",
    "        time.sleep(3)\n",
    "        continue\n",
    "r = open(\"track_features.txt\", \"a\")\n",
    "r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\" _________________________ \"+\"Total Number Of Errors : \"+str(e)+\" _________________________ \"+'\\n')\n",
    "r.close()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/artist_features.csv','a')\n",
    "e=0\n",
    "for i in tqdm(range(0,len(a_uri),50)):\n",
    "    try:\n",
    "        artist_features = sp.artists(a_uri[i:i+50])\n",
    "        for x in range(50):\n",
    "            artist_df=pd.DataFrame([a_uri[i+x]])\n",
    "            artist_pop = artist_features['artists'][x][\"popularity\"]\n",
    "            artist_genres = artist_features['artists'][x][\"genres\"]\n",
    "            artist_df[\"artist_pop\"] = artist_pop\n",
    "            if artist_genres: \n",
    "                artist_df[\"genres\"] = \" \".join([re.sub(' ','_',i) for i in artist_genres])\n",
    "            else:\n",
    "              artist_df[\"genres\"] = \"unknown\"\n",
    "            csv_data = artist_df.to_csv(header=False,index=False)\n",
    "            f.write(csv_data)\n",
    "    except Exception as error:\n",
    "        e+=1\n",
    "        r = open(\"artist_features.txt\", \"a\")\n",
    "        r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\": \"+str(error)+'\\n')\n",
    "        r.close()\n",
    "        time.sleep(3)\n",
    "        continue\n",
    "r = open(\"artist_features.txt\", \"a\")\n",
    "r.write(datetime.datetime.now().strftime(\"%d.%b %Y %H:%M:%S\")+\" _________________________ \"+\"Total Number Of Errors : \"+str(e)+\" _________________________ \"+'\\n')\n",
    "r.close()\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "e246d2215c418239c9316a1ebf2d8abb44dc50b2e5b0e29defd87143398aa387"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}