GUBRA_output_vol_Dan_Excel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "48c9d2b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing all libraries needed\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8e8990e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stanting timming process\n",
    "start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "768a5c8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the main directory\n",
    "main_directory = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b764c18a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# subfolder_name = [f.name for f in os.scandir(main_directory) if f.is_dir()]\n",
    "subfolder_with_path = [\"\" for i in range(4)]\n",
    "subfolder_with_path[0] = main_directory + '/'+'01Activity_files'\n",
    "subfolder_with_path[1] = main_directory + '/'+'02Volume_files'\n",
    "subfolder_with_path[2] = main_directory + '/'+'03Volume_in_tissue_files'\n",
    "subfolder_with_path[3] = main_directory + '/'+'zzz_input_and_reference_files'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7927d0cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################################\n",
    "# Importing information from \"samples overview\", \"intensisties template\" and \"brain gubra map\"\n",
    "###############################################################################################\n",
    "\n",
    "# Changing directory to the \"zzz_input_and_reference_files\" folder\n",
    "os.chdir(subfolder_with_path[-1])\n",
    "\n",
    "\n",
    "# Reading \"sample_overview.csv\" file\n",
    "samples_overview = pd.read_csv(\"sample_overview.csv\", \n",
    "                               dtype={\"SAMPLE\": int,\n",
    "                                      \"hemisphere\": \"string\",\"marker\": \"string\",\n",
    "                                      \"TX_GROUP\": \"string\"})\n",
    "\n",
    "# Reading \"gubra_intensities_template.csv\" file\n",
    "gubra_intensities_template = pd.read_csv(\"gubra_intensities_template.csv\",\n",
    "                                   dtype={\"IDPath\": \"string\", \n",
    "                                          \"LabelID\": int, \n",
    "                                          \"raw\": \"string\", \n",
    "                                          \"LabelAbrv\": \"string\"})\n",
    "# Reading \"brain_allen_map.csv\" file\n",
    "brain_gubra_map = pd.read_csv(\"brain_gubra_map_fixed.csv\", \n",
    "                               dtype={\"IDPath\": \"string\", \"LabelAbrv\": \"string\",\n",
    "                                      \"raw\": \"string\",\"allen_1\": \"string\",\n",
    "                                      \"allen_2\": \"string\",\"allen_3\": \"string\",\n",
    "                                      \"allen_4\": \"string\",\"allen_5\": \"string\",\n",
    "                                      \"allen_6\": \"string\",\"allen_7\": \"string\",\n",
    "                                      \"fine\": \"string\",\"medium\": \"string\",\n",
    "                                      \"coarse\": \"string\", \"all\":\"string\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dcf1cc23",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving drugs and markers' names from the \"sample_overview\" file\n",
    "\n",
    "drugs = samples_overview['TX_GROUP'].unique()\n",
    "markers = samples_overview['marker'].unique()\n",
    "\n",
    "d = 0\n",
    "m = 0\n",
    "counters_names = []\n",
    "while (m < len(markers)):\n",
    "    while (d < len(drugs)):\n",
    "        counters_names.append(drugs[d] + '-' + markers[m])\n",
    "        d += 1\n",
    "    m += 1\n",
    "    d = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "10ad342d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"02Volume_files\" folder\n",
    "os.chdir(subfolder_with_path[1])\n",
    "\n",
    "#Counting number of files on \"02Volume_files\" folder\n",
    "list = os.listdir(subfolder_with_path[1]) \n",
    "number_files = len(list)\n",
    "number_volume_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4192ae1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "volume_data = [] #This array will contain df as elements\n",
    "\n",
    "#Staring counters\n",
    "\n",
    "i = 0 #General loop counter\n",
    "\n",
    "#Zeroing the counters \n",
    "counters = np.zeros(len(counters_names))\n",
    "\n",
    "#Creating counters array to contain sample's names as elements\n",
    "counters_samples = [[] for j in range(len(counters_names))]\n",
    "\n",
    "while i < number_files:\n",
    "\n",
    "    ##############################################\n",
    "    # Processing each file\n",
    "    ##############################################\n",
    "    \n",
    "    file = list[i]\n",
    "    # Saving sample's number\n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "\n",
    "    # Reading file's data\n",
    "    header = [\"PIXEL_COUNT\"]\n",
    "    pixels_count_volume = pd.read_csv(file, names = header)\n",
    "    pixels_count_volume.insert(0, column = 'INTENSITY', value = range(0, len(pixels_count_volume))) \n",
    "\n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = gubra_intensities_template.merge(pixels_count_volume, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    volume_label_name = 'VOLUME'\n",
    "    output = output.rename(columns = {'PIXEL_COUNT':volume_label_name})\n",
    "    output['VOLUME'] = output['VOLUME'].div(1000000).round(7)\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    if hemisphere[0] == 'right':\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "    \n",
    "\n",
    "    \n",
    "    # Merging activity and sample overview data-frames\n",
    "    output= pd.merge(output,samples_overview[['SAMPLE',\n",
    "                                             'hemisphere','marker',\n",
    "                                             'TX_GROUP']],on='SAMPLE',how='inner')\n",
    "    \n",
    "    #Adding a column with the sample's nickname\n",
    "    counters_index = counters_names.index(output['TX_GROUP'].iloc[0] + '-' + output['marker'].iloc[0])\n",
    "    if output['SAMPLE'].iloc[0] not in counters_samples[counters_index]:\n",
    "        counters[counters_index] += 1\n",
    "        counters_samples[counters_index].append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'v-' + output['TX_GROUP'].iloc[0] + '-' + str(int(counters[counters_index]))\n",
    "\n",
    "    nickname_column = np.empty(len(output))    \n",
    "    output['nickname'] = nickname_column\n",
    "    output['nickname'] = nickname\n",
    "    \n",
    "    # In case the sample has both hemispheres\n",
    "    if hemisphere[0] == 'both':\n",
    "        output['nickname'] = output['nickname'] + '-' + output[\"LabelAbrv\"].str[0]\n",
    "        \n",
    "    \n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"LabelAbrv\"] = output[\"LabelAbrv\"].str.replace('L','',1)\n",
    "    output[\"LabelAbrv\"] = output[\"LabelAbrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    volume_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframe\n",
    "volume_data = pd.concat(volume_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7c5c361e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IDPath</th>\n",
       "      <th>LabelID</th>\n",
       "      <th>raw</th>\n",
       "      <th>LabelAbrv</th>\n",
       "      <th>VOLUME</th>\n",
       "      <th>SAMPLE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/997/8/567/623/477/278/23/</td>\n",
       "      <td>23</td>\n",
       "      <td>right_Anterior_amygdalar_area</td>\n",
       "      <td>RAAA</td>\n",
       "      <td>0.125990</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/997/8/567/688/695/315/31/39/935/</td>\n",
       "      <td>935</td>\n",
       "      <td>right_Anterior_cingulate_area_dorsal_part_layer_1</td>\n",
       "      <td>RACAd1</td>\n",
       "      <td>0.149792</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/997/8/567/688/695/315/31/39/211/</td>\n",
       "      <td>211</td>\n",
       "      <td>right_Anterior_cingulate_area_dorsal_part_laye...</td>\n",
       "      <td>RACAd2/3</td>\n",
       "      <td>0.155390</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/997/8/567/688/695/315/31/39/1015/</td>\n",
       "      <td>1015</td>\n",
       "      <td>right_Anterior_cingulate_area_dorsal_part_layer_5</td>\n",
       "      <td>RACAd5</td>\n",
       "      <td>0.329918</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/997/8/567/688/695/315/31/39/919/</td>\n",
       "      <td>919</td>\n",
       "      <td>right_Anterior_cingulate_area_dorsal_part_laye...</td>\n",
       "      <td>RACAd6a</td>\n",
       "      <td>0.192198</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1180</th>\n",
       "      <td>/997/1009/967/933/413/</td>\n",
       "      <td>20413</td>\n",
       "      <td>left_vestibular_nerve</td>\n",
       "      <td>LvVIIIn</td>\n",
       "      <td>0.054029</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1181</th>\n",
       "      <td>/997/8/343/1065/354/370/765/</td>\n",
       "      <td>20765</td>\n",
       "      <td>left_Nucleus_x</td>\n",
       "      <td>Lx</td>\n",
       "      <td>0.010924</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>/997/8/343/1065/354/370/773/</td>\n",
       "      <td>20773</td>\n",
       "      <td>left_Hypoglossal_nucleus</td>\n",
       "      <td>LXII</td>\n",
       "      <td>0.079108</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1183</th>\n",
       "      <td>/997/8/343/1065/354/370/781/</td>\n",
       "      <td>20781</td>\n",
       "      <td>left_Nucleus_y</td>\n",
       "      <td>Ly</td>\n",
       "      <td>0.003807</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1184</th>\n",
       "      <td>/997/8/343/1129/1097/290/797/</td>\n",
       "      <td>20797</td>\n",
       "      <td>left_Zona_incerta</td>\n",
       "      <td>LZI</td>\n",
       "      <td>0.400144</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1185 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  IDPath  LabelID  \\\n",
       "0             /997/8/567/623/477/278/23/       23   \n",
       "1      /997/8/567/688/695/315/31/39/935/      935   \n",
       "2      /997/8/567/688/695/315/31/39/211/      211   \n",
       "3     /997/8/567/688/695/315/31/39/1015/     1015   \n",
       "4      /997/8/567/688/695/315/31/39/919/      919   \n",
       "...                                  ...      ...   \n",
       "1180              /997/1009/967/933/413/    20413   \n",
       "1181        /997/8/343/1065/354/370/765/    20765   \n",
       "1182        /997/8/343/1065/354/370/773/    20773   \n",
       "1183        /997/8/343/1065/354/370/781/    20781   \n",
       "1184       /997/8/343/1129/1097/290/797/    20797   \n",
       "\n",
       "                                                    raw LabelAbrv    VOLUME  \\\n",
       "0                         right_Anterior_amygdalar_area      RAAA  0.125990   \n",
       "1     right_Anterior_cingulate_area_dorsal_part_layer_1    RACAd1  0.149792   \n",
       "2     right_Anterior_cingulate_area_dorsal_part_laye...  RACAd2/3  0.155390   \n",
       "3     right_Anterior_cingulate_area_dorsal_part_layer_5    RACAd5  0.329918   \n",
       "4     right_Anterior_cingulate_area_dorsal_part_laye...   RACAd6a  0.192198   \n",
       "...                                                 ...       ...       ...   \n",
       "1180                              left_vestibular_nerve   LvVIIIn  0.054029   \n",
       "1181                                     left_Nucleus_x        Lx  0.010924   \n",
       "1182                           left_Hypoglossal_nucleus      LXII  0.079108   \n",
       "1183                                     left_Nucleus_y        Ly  0.003807   \n",
       "1184                                  left_Zona_incerta       LZI  0.400144   \n",
       "\n",
       "      SAMPLE  \n",
       "0        1.0  \n",
       "1        1.0  \n",
       "2        1.0  \n",
       "3        1.0  \n",
       "4        1.0  \n",
       "...      ...  \n",
       "1180     1.0  \n",
       "1181     1.0  \n",
       "1182     1.0  \n",
       "1183     1.0  \n",
       "1184     1.0  \n",
       "\n",
       "[1185 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36b91efc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the main path\n",
    "os.chdir(main_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49c6bccf",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "########Creating output for Dan's Excel analysis\n",
    "\n",
    "\n",
    "#Chaging SAMPLE elements to integer and then to string when adding the letter \"S\"\n",
    "volume_data['SAMPLE_STR'] = volume_data['SAMPLE'].astype(int)\n",
    "volume_data['SAMPLE_STR'] = volume_data['SAMPLE_STR'].astype(str)\n",
    "volume_data['SAMPLE_STR'] = 'S' + volume_data['SAMPLE_STR']\n",
    "\n",
    "#Pivoting dataframe to have sample as columns\n",
    "pivoted_volume_Dan = pd.pivot_table(volume_data, values='VOLUME', \n",
    "                               index=['raw','hemisphere',\n",
    "                                      'marker','SAMPLE'],\n",
    "                                      columns=['SAMPLE_STR','nickname'], aggfunc=np.sum)\n",
    "\n",
    "#Restarting index to activate'hemisphere','marker' as columns\n",
    "pivoted_volume_Dan = pivoted_volume_Dan.reset_index()\n",
    "\n",
    "#Adding GUBRA map columns\n",
    "gubra_and_volume_Dan = pd.merge(brain_gubra_map,pivoted_volume_Dan,on='raw',how='right')\n",
    "\n",
    "#Editing output\n",
    "gubra_and_volume_Dan.rename(columns={('SAMPLE', ''): 'SAMPLE',\n",
    "                                       ('hemisphere', ''): 'hemisphere',\n",
    "                                       ('marker', ''): 'marker',\n",
    "                                       'raw': 'LabelName'}, inplace=True)\n",
    "gubra_and_volume_Dan = gubra_and_volume_Dan.drop(columns=['allen_1', 'allen_2',\n",
    "                                     'allen_3', 'allen_4',\n",
    "                                     'allen_5', 'allen_6',\n",
    "                                     'allen_7', 'fine',\n",
    "                                     'medium', 'coarse','all',\n",
    "                                    'SAMPLE', 'hemisphere','marker'])\n",
    "gubra_and_volume_Dan = gubra_and_volume_Dan.groupby('IDPath').apply(lambda x: x\\\n",
    "                                                                        .fillna(method='ffill')\\\n",
    "                                                                        .fillna(method='bfill')\\\n",
    "                                                                        .drop_duplicates()\\\n",
    "                                                                        .reset_index(drop=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5aa4107",
   "metadata": {},
   "outputs": [],
   "source": [
    "#editing columns names from ('sample43', 'psilocybin') to S43_psilocybin\n",
    "columns_name_array = gubra_and_volume_Dan.columns.values\n",
    "#dividing into two arrays, one with str and one with tuple\n",
    "#\"cna\" = column_name+array\n",
    "number_of_cols = columns_name_array.size\n",
    "cna_1 = columns_name_array[0:3]\n",
    "cna_2 = columns_name_array[-(number_of_cols - 3):]\n",
    "str = ''\n",
    "for item in cna_2:\n",
    "    str = str +'_'.join(item) + ' ' \n",
    "str = str.split(' ')[:-1]\n",
    "new_columns_name = np.concatenate([cna_1, str]) \n",
    "new_columns_name\n",
    "#Asingning new column names to the dataframe\n",
    "gubra_and_volume_Dan = gubra_and_volume_Dan.set_axis(new_columns_name, axis=1, inplace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57b679af",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Ordering columns so all saline samples are fisrt\n",
    "cols = gubra_and_volume_Dan.columns.tolist()\n",
    "cols_first_3 = cna_1\n",
    "cols_samples = cols[-(number_of_cols - 3):]\n",
    "\n",
    "def w_len(e):\n",
    "    len_e = len(e)\n",
    "    if operator.contains(e, \"aline\"):\n",
    "        len_e = 1\n",
    "    return len_e\n",
    "\n",
    "cols_samples.sort(key=w_len)\n",
    "new_cols = np.concatenate((cols_first_3, cols_samples), axis=None)\n",
    "\n",
    "gubra_and_volume_Dan = gubra_and_volume_Dan[new_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1383a46a",
   "metadata": {},
   "outputs": [],
   "source": [
    "gubra_and_volume_Dan.to_csv('GUBRA_output_vol_Dan_Excel.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6005378b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Ending timming process\n",
    "end = time.time()\n",
    "\n",
    "# Printing how many files were processed and how much time the process took\n",
    "files_processed = number_activity_files = number_volume_files\n",
    "print(files_processed,' Files processed ')\n",
    "print(' ')\n",
    "print(' Execution time:', round((end - start),2), 'seconds') "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}