heirarclus_and_Kmeans_final aspect weights_Commented.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":22582,"status":"ok","timestamp":1670112350872,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"6SPNKCaPc5wQ","outputId":"87003dcf-6d61-4a1f-a0c5-e5c36e0f175d"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":138,"status":"ok","timestamp":1670112390017,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"Q8P-8wPxc3uU"},"outputs":[],"source":["import os \n","os.chdir(\"/content/drive/MyDrive/ML/Project/\")"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"elapsed":17,"status":"ok","timestamp":1670112395149,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"HoxMP3sQdLjf","outputId":"c946cfcf-2095-4109-ff6b-1d2b485cfd13"},"outputs":[],"source":["pwd"]},{"cell_type":"code","execution_count":27,"metadata":{"executionInfo":{"elapsed":254,"status":"ok","timestamp":1670112442102,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"nLt7Jf1pdN0M"},"outputs":[],"source":["import pickle\n","#read the pickle file into a list\n","with open('Aspects.pickle', 'rb') as handle:\n","    Aspects = pickle.load(handle)"]},{"cell_type":"code","execution_count":28,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":12,"status":"ok","timestamp":1670112453655,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"plzqD7STdY2u","outputId":"54f0df95-3512-4cbe-ad15-90a61dfdc529"},"outputs":[{"data":{"text/plain":["18866"]},"execution_count":28,"metadata":{},"output_type":"execute_result"}],"source":["len(Aspects)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"VT8yFn_JcVOK","outputId":"0e787631-2e9e-4819-da43-039f6417512c"},"outputs":[],"source":["import pandas as pd\n","import re\n","df5 = pd.read_csv(\"Hotel_Reviews_cleaned_nouns_and_adjs.csv\")\n","print(df5.head())\n","list_of_nouns=[]\n","for i in range(df5.shape[0]):\n","    list_of_nouns.extend(\n","        re.sub(r'[^a-zA-Z0-9\\s]', '', df5['noun_list'][i]).split())\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qVEfOpojcVOO","outputId":"83bedd53-c139-4867-faad-079dbc4e42cc"},"outputs":[],"source":["list_of_nouns_wdp=list((set(list_of_nouns)))\n","#sorting list of nouns_wdp in alphabetical order\n","list_of_nouns_wdp.sort()\n","print(len(list_of_nouns_wdp))\n","print(list_of_nouns_wdp[0:50])\n","final_list = []\n","for i in list_of_nouns_wdp:\n","    if not(bool(re.search(r'\\d', i)) or len(i)<3 or i.count(i[0]) == len(i)):\n","        final_list.append(i)\n","print(len(final_list))\n","print(final_list[0:50])"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":116,"status":"ok","timestamp":1670112635119,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"CeUPeVceeBuo","outputId":"6d6f7e6d-1527-488f-f665-07afeaf1e596"},"outputs":[],"source":["Aspects.sort()\n","Aspects[:50]"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import nltk\n","# nltk.download('wordnet')\n","nltk.download('omw-1.4')\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#lemmatise the list of nouns\n","from nltk.stem import WordNetLemmatizer\n","lemmatizer = WordNetLemmatizer()\n","lemmatized_list = []\n","for i in Aspects:\n","    lemmatized_list.append(lemmatizer.lemmatize(i))\n","print(len(lemmatized_list))\n","print(len(Aspects))\n"]},{"cell_type":"code","execution_count":29,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["18866\n","[ 3.30089748e-01  5.39969914e-02  1.95969987e+00 -2.17744000e-02\n"," -3.75243664e-01  5.51731467e-01 -4.69461173e-01  3.58215153e-01\n"," -6.29076362e-01 -7.77348205e-02 -8.75175416e-01 -1.85526147e-01\n","  4.95407343e-01  4.60092217e-01  1.15339525e-01 -2.33357340e-01\n"," -1.26124692e+00 -4.60850894e-01  6.92168474e-01 -7.61362195e-01\n"," -1.41604897e-02  2.13784039e-01 -1.95404030e-02 -5.56954205e-01\n"," -7.19833970e-01 -8.10428143e-01 -3.58757675e-01 -1.06132698e+00\n","  4.13742810e-01  1.92816913e-01 -3.77652526e-01 -9.74392176e-01\n","  6.85736656e-01 -3.04110050e-01 -1.04661167e+00 -4.60280389e-01\n"," -1.64322108e-02  2.19158888e-01  2.33379126e-01  4.15027678e-01\n"," -7.27472231e-02 -7.94542432e-01  1.32710826e+00  5.13561487e-01\n"," -1.24228263e+00 -5.78514636e-01  7.05976963e-01  5.81623018e-01\n","  1.14856541e-01 -3.50018680e-01 -7.26326466e-01 -5.48522532e-01\n","  5.98771036e-01  9.51106787e-01 -1.06407627e-01  5.59503913e-01\n"," -3.89826715e-01 -8.15315068e-01 -2.26561412e-01  4.11355495e-01\n"," -4.03983369e-02 -6.94601357e-01  1.06437755e+00  3.62206936e-01\n"," -2.06093527e-02  5.49815655e-01  6.34932637e-01 -8.81038666e-01\n"," -1.09454244e-01 -4.49249983e-01  1.01276004e+00 -7.57807910e-01\n","  2.94685066e-01  1.02443218e+00 -1.56618047e+00 -7.15753973e-01\n","  5.75634241e-01  6.61127925e-01  5.90688214e-02  3.45234275e-01\n"," -3.34531099e-01  1.54727459e-01  9.86847401e-01  5.81415653e-01\n"," -5.53846918e-02  1.58198282e-01  5.53711414e-01 -2.13964373e-01\n"," -1.37558937e+00  5.50787330e-01  2.85314620e-01  4.25906867e-01\n","  1.08169532e+00 -3.72123897e-01  2.17829123e-01 -2.31595814e-01\n","  8.56578350e-02 -3.61640975e-02  3.12470138e-01 -4.63746935e-01\n"," -7.19592035e-01  7.10861444e-01  3.48590128e-03 -2.31584460e-01\n","  5.02379060e-01  1.20927617e-02 -1.30814925e-01  1.27592608e-01\n","  8.63191038e-02  3.89786243e-01  1.60504445e-01 -4.01388526e-01\n","  4.70330209e-01  3.11555624e-01 -4.98058498e-01  4.93796349e-01\n"," -7.54850388e-01 -4.79963183e-01  1.16341859e-01  9.33818221e-01\n","  5.71824968e-01  8.26698899e-01 -8.39346647e-02 -1.14381027e+00\n","  3.89471024e-01 -8.11472356e-01  7.15934634e-01  7.24925041e-01\n"," -8.57811213e-01  4.01672065e-01 -9.41801816e-02  1.35565609e-01\n","  4.96730447e-01  7.10465014e-01 -2.15752125e-01 -2.39646643e-01\n","  3.36324126e-01 -8.12227488e-01 -3.76706839e-01  7.55933285e-01\n"," -6.83721781e-01  6.88255250e-01  5.68434298e-01  2.29483694e-01\n"," -1.21691525e-01 -8.30634594e-01 -3.83595884e-01 -2.86444306e-01\n"," -6.73352361e-01 -1.08351484e-01 -6.22591555e-01  1.31472617e-01\n"," -4.95736867e-01 -9.65833008e-01 -7.64538497e-02  5.27198732e-01\n","  4.79483664e-01 -3.72095220e-02  5.99678569e-02  1.03840053e-01\n","  5.56912541e-01  1.32782900e+00 -9.42009032e-01 -2.39074215e-01\n","  3.04666966e-01  1.99369282e-01 -5.07858470e-02  5.78822434e-01\n","  4.71762717e-01  1.08687520e-01  5.95701993e-01 -6.34408236e-01\n"," -2.99599081e-01  2.15699151e-01  3.46599892e-02 -8.86480063e-02\n","  5.74738622e-01  3.13580930e-01  7.39108562e-01 -1.68251097e-01\n"," -1.76279582e-02  4.96808589e-01 -1.90042332e-01 -8.36077482e-02\n"," -4.06381249e-01 -1.12807226e+00 -4.11219358e-01  5.31814247e-02\n","  1.89388677e-01 -6.23981297e-01  3.53540838e-01 -4.59412843e-01\n"," -9.91184533e-01 -7.21547484e-01 -5.95545888e-01  4.65371102e-01\n","  6.70933366e-01  6.78656518e-01 -3.35702211e-01  4.20890540e-01\n","  8.55267406e-01 -2.83184946e-01  1.91983819e-01  2.22093731e-01\n","  3.13371092e-01  8.77977669e-01 -2.17558771e-01  6.47381604e-01\n","  2.64530540e-01 -6.60589933e-01 -5.89600444e-01  2.84395695e-01\n","  9.02504086e-01  3.13581318e-01 -4.57425684e-01 -1.71455257e-02\n","  9.57067251e-01 -7.66753614e-01 -1.38659906e+00 -3.53492171e-01\n","  1.39669442e+00  8.77342343e-01  9.00805831e-01  1.48545611e+00\n"," -4.70022529e-01  5.75462103e-01  9.43728924e-01  1.75508723e-01\n"," -6.96861506e-01  1.19142219e-01  8.14751744e-01  3.89936328e-01\n","  3.28991413e-01  4.00578082e-01  2.76851654e-01 -2.77708560e-01\n","  1.68552861e-01 -5.41037321e-01 -7.09947705e-01  6.39666438e-01\n"," -5.49435914e-02  1.06225514e+00  5.54481566e-01  2.23270029e-01\n"," -9.60722387e-01 -6.25136673e-01  9.60039735e-01 -7.42923141e-01\n"," -3.32038134e-01  6.92808747e-01 -3.65790159e-01 -1.03247154e+00\n"," -4.10089642e-01 -4.26522583e-01 -1.95268050e-01  1.26549974e-01\n"," -6.17405213e-02  4.93611723e-01  2.76814222e-01 -1.81473255e-01\n","  8.63498449e-02  1.81720898e-01 -1.03909087e+00 -2.17314184e-01\n","  7.10958540e-02  5.77335134e-02 -8.55584741e-01  3.83870363e-01\n"," -3.56503397e-01  6.00008249e-01 -1.16271451e-01 -2.05349535e-01\n"," -8.44381034e-01 -6.21994399e-03  2.07578242e-02 -5.37894845e-01\n","  6.69940352e-01 -4.32771176e-01  1.69161752e-01 -1.49673581e-01\n","  1.58810318e-01  9.71350670e-02 -1.05413401e+00  1.65641278e-01\n","  3.06916833e-01 -3.27307016e-01 -4.63908799e-02  3.58032405e-01\n"," -3.03876579e-01 -1.07769418e+00 -5.33195078e-01 -9.92000028e-02\n","  3.02377105e-01 -9.33595777e-01  1.33112931e+00 -6.84566855e-01\n","  2.68313617e-01  2.78776824e-01  1.17154673e-01  1.73480853e-01\n"," -6.18683517e-01 -1.68408692e-01 -3.99867475e-01 -8.74767184e-01\n"," -3.31695110e-01 -7.17350721e-01  1.58560947e-01 -1.01270664e+00\n"," -1.49689448e+00 -3.79257321e-01 -4.47293133e-01 -2.31190607e-01\n","  6.50404334e-01 -6.09032512e-01  5.28939784e-01  5.62004507e-01\n"," -1.45531386e-01  5.24806798e-01 -9.84460041e-02 -3.98351729e-01\n"," -7.95092523e-01  5.49374878e-01  3.41908671e-02  1.05085957e+00\n"," -1.36022761e-01  4.44321454e-01  1.19145900e-01  6.40382767e-02\n"," -8.39830637e-01  2.03300640e-02  9.35075879e-01 -6.68063983e-02\n"," -9.14876699e-01  1.43467486e-01  3.95800382e-01  2.74376631e-01\n"," -5.95323801e-01  1.30845413e-01  7.85579205e-01  1.25215614e+00\n","  1.51379868e-01  1.75705940e-01 -7.90251140e-03 -1.21475124e+00\n","  1.06509900e+00 -1.12763429e+00  1.15772530e-01 -9.58069563e-01\n","  7.46421635e-01  5.06033659e-01  2.47139767e-01 -2.05610365e-01\n","  1.20115772e-01 -2.64502645e-01 -2.28570604e+00 -4.95772988e-01\n","  3.42531413e-01 -1.48944616e-01  3.68358195e-01  2.21327960e-01\n","  6.75839186e-01 -6.30774498e-02 -8.74306858e-01  3.49338710e-01\n","  3.90837729e-01  1.53512061e-01  3.51660132e-01 -5.03504932e-01\n","  8.88653576e-01  8.75764608e-01  2.29452446e-01 -1.10256398e+00\n"," -6.75883532e-01  1.06297396e-01 -8.85445237e-01  3.58547568e-01\n"," -4.33931917e-01 -7.13146806e-01 -5.32138497e-02 -4.82177019e-01\n","  6.31899416e-01 -5.96099377e-01 -7.32162520e-02 -9.88300562e-01\n","  5.37693761e-02  5.53187251e-01  4.24945623e-01  7.68309534e-01\n","  1.10081851e+00 -3.33654881e-01  8.96272063e-01 -6.43277727e-03\n"," -2.51748532e-01 -4.53601688e-01  3.80097069e-02 -1.14138794e+00\n","  4.79647517e-01  6.41668797e-01 -4.46521461e-01  1.82308823e-01\n","  1.78346813e-01  5.92127562e-01  2.68566221e-01  5.28235912e-01\n","  3.08041394e-01 -1.62383288e-01  6.58918858e-01 -5.95084310e-01\n"," -5.96745551e-01 -1.37689963e-01 -4.09523845e-01 -1.13364175e-01\n","  2.97539145e-01 -2.72331610e-02 -1.09472609e+00 -2.04228327e-01\n"," -1.86551377e-01 -1.09998631e+00  1.18602371e+00  1.49805531e-01\n"," -6.03577256e-01 -7.18686223e-01 -4.06844318e-01  8.18334103e-01\n"," -3.99802774e-01  8.72235656e-01  1.32379398e-01 -1.30875528e-01\n"," -3.84321272e-01 -8.42290521e-01 -1.51586390e+00  2.74458885e-01\n"," -1.33305895e+00  7.15097070e-01 -3.80754143e-01 -3.92824769e-01\n"," -3.85791540e-01  4.74494040e-01 -1.74460411e-01  2.21059635e-01\n","  8.00476670e-01 -4.63869691e-01  4.18296307e-01  5.05715609e-01\n"," -3.19000423e-01 -1.57257304e-01  3.16857874e-01  4.04766142e-01\n","  2.17461139e-01 -7.12860405e-01 -1.82511553e-01  7.65980601e-01\n","  6.83961749e-01  7.59990588e-02 -7.00511098e-01  2.31864020e-01\n","  6.97779119e-01  6.20944612e-03  8.46201330e-02 -1.06996512e+00\n"," -1.13587284e+00  7.21250474e-02 -2.06704736e-01 -5.72302997e-01\n","  2.12364733e-01 -1.00494832e-01 -1.11597466e+00  2.32326865e-01\n"," -6.80348873e-02  4.32557762e-01 -2.91914731e-01 -5.34689963e-01\n"," -5.95509827e-01 -2.06427112e-01  3.49513948e-01 -4.40589547e-01\n"," -1.10938263e+00  6.83129787e-01 -3.76268327e-01  4.38352734e-01\n","  7.46968806e-01  2.37611666e-01 -1.78215772e-01  1.01743376e+00\n","  5.39186336e-02  1.04051423e+00  1.02126062e+00  2.44337007e-01\n","  5.42433619e-01  5.08340716e-01 -8.52127671e-01  1.34587884e-01\n"," -1.21292722e+00 -6.70752048e-01 -3.86627197e-01 -4.99565780e-01\n"," -8.33356619e-01 -1.95333987e-01  1.72766283e-01  5.85832894e-01\n"," -5.04400671e-01  5.65451264e-01  8.75864625e-02 -2.57615885e-03\n"," -3.36870626e-02  1.03614652e+00 -1.17849112e+00  7.42493093e-01\n","  4.61186588e-01  8.43577027e-01 -1.02818108e+00 -9.09607559e-02\n"," -1.22472249e-01 -6.37813628e-01  3.45103383e-01 -1.42388433e-01\n"," -6.11523032e-01 -3.62434179e-01 -2.40895718e-01 -1.68820396e-01\n"," -8.11179996e-01 -1.50225163e-01  3.55461091e-01 -1.55043185e+00\n","  4.28187288e-02  6.45097017e-01  5.12062132e-01  1.52224943e-01\n","  4.62091208e-01 -2.54746199e-01 -7.11102664e-01 -4.22824711e-01\n"," -8.05720448e-01 -2.95465469e-01  4.88003314e-01 -3.81427854e-01\n"," -6.84581220e-01 -5.32784343e-01  5.15455425e-01  9.91636440e-02\n","  6.63943231e-01 -5.75463846e-02  7.32034206e-01 -2.78340369e-01\n"," -1.65151119e+00 -7.00743973e-01  1.25351980e-01 -1.54732406e-01\n"," -1.09689534e+00 -1.23576570e+00 -4.40334857e-01  1.61133900e-01\n"," -4.12033200e-01 -1.78328961e-01 -5.75805843e-01 -8.71240497e-01\n","  3.84896398e-02  9.12587345e-01  1.42023861e-01 -7.02761352e-01\n","  6.69208348e-01  4.37118292e-01 -1.41122378e-03  8.08609605e-01\n"," -5.81499338e-01 -8.12098980e-02  1.75465316e-01  1.63131580e-02\n"," -1.67295158e-01  2.32177868e-01  2.82010585e-01 -1.83793724e-01\n","  6.22099876e-01  2.28234887e+00 -1.21957824e-01  5.39516926e-01\n","  6.35087013e-01  1.12807550e-01 -6.34567261e-01  8.17527950e-01\n"," -3.41706038e-01  3.78005505e-01  3.28958750e-01 -6.69564843e-01\n"," -2.96171904e-01  4.40271348e-01  5.82381725e-01 -7.57616997e-01\n"," -7.46498480e-02 -6.36470169e-02  9.90304112e-01 -1.58894315e-01\n","  2.96793729e-01 -7.78860748e-01  5.48785985e-01 -6.46041989e-01\n"," -1.06299520e-01  1.09308743e+00  4.62860256e-01  7.88026094e-01\n"," -2.23112464e-01  1.08806372e-01 -3.60261917e-01  4.76399004e-01\n","  2.55402982e-01  5.51058650e-01 -1.06329203e+00  2.63666570e-01\n"," -1.21496722e-01  1.12527773e-01 -3.76184672e-01 -1.00963950e+00\n"," -1.74433798e-01  1.08089641e-01 -5.12575090e-01 -1.67814469e+00\n"," -1.29908276e+00  1.01858400e-01 -1.73726320e-01  8.75354558e-02\n","  5.66604316e-01 -2.12418750e-01  1.79094553e-01 -1.93508610e-01\n","  4.46725637e-01  3.59839439e-01 -2.02610344e-01 -4.47415322e-01\n","  3.20385732e-02 -5.65861702e-01  1.07665634e+00  1.74149141e-01\n","  4.69293177e-01 -1.51428580e+00  8.05464149e-01  8.49294603e-01\n"," -6.51036978e-01 -2.56578326e-01  9.54513907e-01  1.31344825e-01\n","  3.15578967e-01 -3.75626981e-02  7.84282684e-01 -9.90673959e-01\n"," -1.35926270e+00 -5.04677594e-01  3.52944523e-01  6.19630098e-01\n","  7.27483153e-01  1.35922658e+00  5.82417965e-01 -1.08708870e+00\n","  1.47051647e-01 -1.81379728e-02  9.73479748e-01  1.12750083e-02\n"," -1.56502910e-02 -2.91003495e-01  5.03809080e-02 -4.82509136e-01\n","  8.74546111e-01 -8.08780417e-02  1.01262724e+00 -1.83177009e-01\n","  4.66015607e-01  3.03224713e-01  3.67797315e-01 -4.08618033e-01\n"," -2.39567533e-01 -7.21988559e-01  1.09057210e-01 -9.20034423e-02\n","  2.92681634e-01 -1.07060146e+00 -3.27984124e-01 -4.18913782e-01\n","  1.54599535e+00 -3.78844440e-01 -1.00006625e-01  9.23083276e-02\n","  1.14187747e-02 -3.16822827e-01  2.04254940e-01 -5.36534011e-01\n"," -2.13325411e-01  9.10012051e-02  3.14291984e-01  2.71592915e-01\n","  6.32416487e-01  1.04123816e-01  1.21922457e+00  3.50375742e-01\n"," -5.00736892e-01 -1.18237972e+00 -9.18521285e-02  1.85388997e-01\n","  3.89019623e-02  1.58307567e-01  1.02468121e+00 -2.81394184e-01\n"," -1.05468377e-01 -6.54010713e-01  6.42601848e-01  3.13471913e-01\n"," -3.49454045e-01 -1.47584260e+00  3.16747427e-01 -4.10334468e-01\n"," -3.32461685e-01  9.82619524e-01 -1.53275877e-01 -1.95284963e-01\n","  2.70259902e-02  6.98711872e-01 -2.40419805e-01  1.63372699e-02\n"," -1.90648422e-01  5.14078438e-01  4.48733598e-01 -3.52248251e-01\n"," -3.88053954e-01  9.24198776e-02 -4.46756482e-01  7.59043694e-01\n"," -5.24761900e-02 -4.16267753e-01  6.80696607e-01  9.99866351e-02\n"," -9.56535220e-01 -7.95851111e-01  8.61472726e-01  9.11700577e-02\n","  6.70481265e-01  6.46344781e-01  5.04890621e-01  1.88277304e-01\n","  6.36620700e-01  1.97484225e-01  3.26661527e-01 -2.78344154e-01\n","  2.23103464e-01  4.16353345e-01  2.73812145e-01 -1.97501376e-01\n"," -8.80089462e-01 -1.28045693e-01  3.85260940e-01 -3.22259843e-01\n"," -5.03552973e-01 -2.08994031e-01  1.85635120e-01 -7.79239163e-02\n","  3.70694518e-01  4.89590541e-02 -1.06098819e+00 -1.66018412e-01\n"," -1.57234192e+00 -2.28518292e-01  2.39695355e-01 -1.52819479e+00\n","  4.76386666e-01  2.40489505e-02 -5.10510325e-01 -2.83050895e-01\n"," -4.26746607e-01  3.55643749e-01 -7.98759460e-01  2.53536820e-01]\n"]}],"source":["#Bert topic modelling on the list of nouns Aspects\n","from sentence_transformers import SentenceTransformer\n","model = SentenceTransformer('bert-base-nli-mean-tokens')\n","embeddings = model.encode(Aspects)\n","print(len(embeddings))\n","print(embeddings[0])"]},{"cell_type":"code","execution_count":30,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[6 5 8 ... 8 8 1]\n"]}],"source":["#Clustering the aspects using KMeans\n","from sklearn.cluster import KMeans\n","import numpy as np\n","num_clusters = 10\n","clustering_model = KMeans(n_clusters=num_clusters)\n","clustering_model.fit(embeddings)\n","cluster_assignment = clustering_model.labels_\n","print(cluster_assignment)\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["clustered_sentences = {i: [] for i in range(num_clusters)}\n","for sentence_id, cluster_id in enumerate(cluster_assignment):\n","    clustered_sentences[cluster_id].append(Aspects[sentence_id])\n","\n","for i, cluster in clustered_sentences.items():\n","    print(\"Cluster \", i+1)\n","    #Write each cluster to a pickle file with filename as the cluster number\n","    with open('cluster'+str(i+1)+'.pickle', 'wb') as handle:\n","        pickle.dump(cluster, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","    print(cluster)\n","    print(\"\")\n","\n"]},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Silhouette score:  0.09305885\n","Calinski Harabasz score:  850.6244248540218\n","Davies Bouldin score:  2.827245415207996\n"]}],"source":["from sklearn.metrics import pairwise_distances\n","pairwise_dist = pairwise_distances(embeddings)\n","\n","#Evaluation of the Kmeans clustering\n","from sklearn.metrics import silhouette_score\n","silhouette_score(pairwise_dist, cluster_assignment)\n","print(\"Silhouette score: \", silhouette_score(embeddings, cluster_assignment))\n","\n","#Evaluation of the Kmeans clustering\n","from sklearn.metrics import calinski_harabasz_score\n","calinski_harabasz_score(embeddings, cluster_assignment)\n","print(\"Calinski Harabasz score: \", calinski_harabasz_score(embeddings, cluster_assignment))\n","\n","#Evaluation of the Kmeans clustering\n","from sklearn.metrics import davies_bouldin_score\n","davies_bouldin_score(embeddings, cluster_assignment)\n","print(\"Davies Bouldin score: \", davies_bouldin_score(embeddings, cluster_assignment))\n","\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#load master dictionary pickle file\n","with open('master_dict.pickle', 'rb') as handle:\n","    master_dict = pickle.load(handle)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#loading cluster pickle files and assign names to clusters\n","cluster1=[]\n","with open('cluster1.pickle', 'rb') as handle:\n","    cluster1 = pickle.load(handle)\n","cluster2=[]\n","with open('cluster2.pickle', 'rb') as handle:\n","    cluster2 = pickle.load(handle)\n","cluster3=[]\n","with open('cluster3.pickle', 'rb') as handle:\n","    cluster3 = pickle.load(handle)\n","cluster4=[]\n","with open('cluster4.pickle', 'rb') as handle:\n","    cluster4 = pickle.load(handle)\n","cluster5=[]\n","with open('cluster5.pickle', 'rb') as handle:\n","    cluster5 = pickle.load(handle)\n","cluster6=[]\n","with open('cluster6.pickle', 'rb') as handle:\n","    cluster6 = pickle.load(handle)\n","cluster7=[]\n","with open('cluster7.pickle', 'rb') as handle:\n","    cluster7 = pickle.load(handle)\n","cluster8=[]\n","with open('cluster8.pickle', 'rb') as handle:\n","    cluster8 = pickle.load(handle)\n","cluster9=[]\n","with open('cluster9.pickle', 'rb') as handle:\n","    cluster9 = pickle.load(handle)\n","cluster10=[]\n","with open('cluster10.pickle', 'rb') as handle:\n","    cluster10 = pickle.load(handle)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#Renaming clusters and combining similar clusters\n","Staff=cluster2\n","Location=cluster10\n","Ambience=cluster6\n","Rooms=cluster5\n","Food=cluster4+cluster7+cluster8\n","Furniture=cluster3\n","Amenities=cluster1\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#writing the renamed clusters to a pickle files\n","with open('Staff.pickle', 'wb') as handle:\n","    pickle.dump(Staff, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","with open('Location.pickle', 'wb') as handle:\n","    pickle.dump(Location, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","with open('Ambience.pickle', 'wb') as handle:\n","    pickle.dump(Ambience, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","with open('Rooms.pickle', 'wb') as handle:\n","    pickle.dump(Rooms, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","with open('Food.pickle', 'wb') as handle:\n","    pickle.dump(Food, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","with open('Furniture.pickle', 'wb') as handle:\n","    pickle.dump(Furniture, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","with open('Amenities.pickle', 'wb') as handle:\n","    pickle.dump(Amenities, handle, protocol=pickle.HIGHEST_PROTOCOL)\n","    "]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import pandas as pd\n","#initilise the data frame with Hotel name, Staff, Location, Ambience, Rooms, Food, Furniture, Amenities,lat, lng as columns\n","df6 = pd.DataFrame(columns=['Hotel_name','Staff','Location','Ambience','Rooms','Food','Furniture','Amenities','lat','lng'])\n","\n","#loop through the master dictionary and assign the values to the data frame if aspect is in clusers\n","for i in master_dict:\n","    df6 = df6.append({'Hotel_name': i,'Staff':0.0,'Location':0.0,'Ambience':0.0,'Rooms':0.0,'Food':0.0,'Furniture':0.0,'Amenities':0.0}, ignore_index=True)\n","    for j in master_dict[i]:\n","        if j in Staff:\n","            df6.loc[df6['Hotel_name'] == i, 'Staff'] += master_dict[i][j]\n","        elif j in Location:\n","            df6.loc[df6['Hotel_name'] == i, 'Location'] += master_dict[i][j]\n","        elif j in Ambience:\n","            df6.loc[df6['Hotel_name'] == i, 'Ambience'] += master_dict[i][j]\n","        elif j in Rooms:\n","            df6.loc[df6['Hotel_name'] == i, 'Rooms'] += master_dict[i][j]\n","        elif j in Food:\n","            df6.loc[df6['Hotel_name'] == i, 'Food'] += master_dict[i][j]\n","        elif j in Furniture:\n","            df6.loc[df6['Hotel_name'] == i, 'Furniture'] += master_dict[i][j]\n","        elif j in Amenities:\n","            df6.loc[df6['Hotel_name'] == i, 'Amenities'] += master_dict[i][j]\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#Randomly assign lat in range (33, 35) and lng in range (-83, -85) to the data frame (Atlanta area)\n","df6['lat'] = np.random.uniform(33, 35, size=len(df6))\n","df6['lng'] = np.random.uniform(-83, -85, size=len(df6))\n","df6['rating'] = np.random.randint(4, 5, size=len(df6))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["df6.head()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#write the data frame to a csv file\n","df6.to_csv('Hotel_name_Aspect_weights.csv', index=False)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":62664,"status":"ok","timestamp":1670112882452,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"rWHw4Z9RexR1","outputId":"de21231d-d78d-4954-ed21-9555f255a0f4"},"outputs":[],"source":["!pip install -U spacy\n","!python -m spacy download en_core_web_lg"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":118,"status":"ok","timestamp":1670112887338,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"iqtrt3-5cVOQ"},"outputs":[],"source":["words_str = ' '.join(Aspects)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3594293,"status":"ok","timestamp":1670116482419,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"-lWS7y3pcVOR","outputId":"0bc51524-af08-4ebb-9f11-841a5fd6e5b2"},"outputs":[],"source":["import spacy\n","import pandas as pd \n","import numpy as np\n","nlp = spacy.load('en_core_web_lg')\n","#words = nlp('breakfast food deserts fruits room hashbrowns vegetarians hair bathroom')\n","words = nlp(words_str)\n","df= pd.DataFrame()\n","#cols = ['breakfast','food','deserts','fruits','room','hashbrowns','vegetarians','hair','bathroom']\n","cols = Aspects\n","sim = dict.fromkeys(words)\n","arr = np.empty((len(cols),len(cols)))\n","print(arr.shape)\n","arr.fill(1)\n","k = 0\n","for i in range(len(cols)-1):\n","    for j in range(i+1,len(cols)):\n","        token1 = words[i]\n","        token2 = words[j]\n","        temp = token1.similarity(token2)\n","        arr[i][j] = temp\n","        arr[j][i] = temp\n","print(arr)\n","df2 = pd.DataFrame(arr,columns=cols, index = cols)\n","print(df2)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":4022,"status":"ok","timestamp":1670117266716,"user":{"displayName":"B Prasanth","userId":"16827775273874832010"},"user_tz":300},"id":"RgubeSYwuW9l"},"outputs":[],"source":["#write the keys to a pickle file\n","import pickle\n","with open('Similarity.pickle', 'wb') as handle:\n","    pickle.dump(arr, handle, protocol=pickle.HIGHEST_PROTOCOL)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#load the keys from a pickle file\n","import numpy as np\n","import pandas as pd\n","import pickle\n","with open('Similarity.pickle', 'rb') as handle:\n","    arr = pickle.load(handle)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["cols = Aspects\n","df2 = pd.DataFrame(arr, columns=cols, index=cols)\n","print(df2)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"unnZk8egcVOS"},"outputs":[],"source":["import seaborn as sns\n","sns.heatmap(round(df2,2), cmap='RdBu', annot=True, \n","            annot_kws={\"size\": 7}, vmin=-1, vmax=1);"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"XO9QcmQ2cVOT","outputId":"58f5c587-9426-44f0-a26b-ddaab0239b36"},"outputs":[],"source":["from scipy.cluster.hierarchy import linkage, dendrogram, fcluster\n","from scipy.spatial.distance import squareform\n","import matplotlib.pyplot as plt\n","plt.figure(figsize=(12,5))\n","dissimilarity = 1 - abs(df2)\n","Z = linkage(squareform(dissimilarity), 'complete')#\n","\n","dendrogram(Z, labels=cols, orientation='top',leaf_rotation=90);"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"8XmJADHBcVOU","outputId":"6e8ef38e-0dfa-4975-d9c1-5f551e235210"},"outputs":[],"source":["threshold = 0.5\n","Numclusters = 500\n","while threshold <1:\n","    labels = fcluster(Z, threshold, criterion='distance')\n","    if(len(np.unique(labels))<Numclusters):\n","        break\n","    threshold+=0.001\n","print(threshold)\n","# Show the cluster\n","labels"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"-njdBdOocVOV","outputId":"8305a39e-e9aa-4fc3-afca-952d78bcdd54"},"outputs":[],"source":["x = []\n","for i in np.unique(labels):\n","    index = np.where(labels == i)\n","    cols = np.array(cols)\n","    x.append(cols[index[0]].tolist())\n","x"]}],"metadata":{"colab":{"provenance":[]},"kernelspec":{"display_name":"Python 3.10.6","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.6"},"orig_nbformat":4,"vscode":{"interpreter":{"hash":"1c78b8c11488983031765fc26fae50f1603d4de4138f229663396b39af78eff5"}}},"nbformat":4,"nbformat_minor":0}