ALLEN_cell_counts_and_region_volumes_090821.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing all libraries needed\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stanting timming process\n",
    "start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the main directory\n",
    "main_directory = os.getcwd()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rreading the name of the subfolder\n",
    "# subfolder_with_path = [f.path for f in os.scandir(main_directory) if f.is_dir()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subfolder_name = [f.name for f in os.scandir(main_directory) if f.is_dir()]\n",
    "subfolder_with_path = [\"\" for i in range(4)]\n",
    "subfolder_with_path[0] = main_directory + '/'+'01Activity_files'\n",
    "subfolder_with_path[1] = main_directory + '/'+'02Volume_files'\n",
    "subfolder_with_path[2] = main_directory + '/'+'03Volume_in_tissue_files'\n",
    "subfolder_with_path[3] = main_directory + '/'+'zzz_input_and_reference_files'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################################\n",
    "# Importing information from \"samples overview\", \"intensisties template\" and \"brain allen map\"\n",
    "###############################################################################################\n",
    "\n",
    "# Changing directory to the \"zzz_input_and_reference_files\" folder\n",
    "os.chdir(subfolder_with_path[-1])\n",
    "\n",
    "\n",
    "# Reading \"sample_overview.csv\" file\n",
    "samples_overview = pd.read_csv(\"sample_overview.csv\", \n",
    "                               dtype={\"MOUSE\": \"string\",\"SAMPLE\": int,\n",
    "                                      \"FILES_HD\": int,\"FILES_BOX\": int,\n",
    "                                      \"hemisphere\": \"string\",\"STAIN\": \"string\",\n",
    "                                      \"TX_GROUP\": \"string\", \n",
    "                                      \"GENDER\":\"string\", \n",
    "                                      \"CONTEXT\":\"string\"})\n",
    "\n",
    "# Reading \"intensities_template.csv\" file\n",
    "intensities_template = pd.read_csv(\"intensities_template.csv\",\n",
    "                                   dtype={\"IDPath\": \"string\", \n",
    "                                          \"LabelID\": int, \n",
    "                                          \"raw\": \"string\", \n",
    "                                          \"labelabrv\": \"string\"})\n",
    "# Reading \"brain_allen_map.csv\" file\n",
    "brain_allen_map = pd.read_csv(\"brain_allen_map.csv\", \n",
    "                               dtype={\"raw\": \"string\",\"allen_1\": \"string\",\n",
    "                                      \"allen_2\": \"string\",\"allen_3\": \"string\",\n",
    "                                      \"allen_4\": \"string\",\"allen_5\": \"string\",\n",
    "                                      \"allen_6\": \"string\",\"allen_7\": \"string\",\n",
    "                                      \"fine\": \"string\",\"medium\": \"string\",\n",
    "                                      \"coarse\": \"string\", \"all\":\"string\"})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"02Volume_files\" folder\n",
    "os.chdir(subfolder_with_path[1])\n",
    "\n",
    "#Counting number of files on \"02Volume_files\" folder\n",
    "list = os.listdir(subfolder_with_path[1]) \n",
    "number_files = len(list)\n",
    "number_volume_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "volume_data = [] #This array will contain df as elements\n",
    "\n",
    "i = 0\n",
    "while i < number_files:\n",
    "\n",
    "    ##############################################\n",
    "    # Processing each file\n",
    "    ##############################################\n",
    "    \n",
    "    file = list[i]\n",
    "    # Saving sample's number\n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "\n",
    "    # Reading file's data\n",
    "    header = [\"PIXEL_COUNT\"]\n",
    "    pixels_count_volume = pd.read_csv(file, names = header)\n",
    "    pixels_count_volume.insert(0, column = 'INTENSITY', value = range(0, len(pixels_count_volume))) \n",
    "\n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = intensities_template.merge(pixels_count_volume, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    volume_label_name = 'VOLUME'\n",
    "    output = output.rename(columns = {'PIXEL_COUNT':volume_label_name})\n",
    "    output['VOLUME'] = output['VOLUME'].div(1000000).round(7)\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    else:\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "   \n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "    \n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('L','',1)\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    volume_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframe\n",
    "volume_data = pd.concat(volume_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"03Volume_in_tissue_files\" folder\n",
    "os.chdir(subfolder_with_path[2])\n",
    "\n",
    "#Counting number of files on \"03Volume_in_tissue_files\" folder\n",
    "list = os.listdir(subfolder_with_path[2]) \n",
    "number_files = len(list)\n",
    "number_volintissue_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "volume_intissue_data = [] #This array will contain df as elements\n",
    "\n",
    "#Staring counters\n",
    "\n",
    "i = 0 #General loop counter\n",
    "\n",
    "sal_tdt = 0 # Counter of samples where Saline was used and tdTomato expressed\n",
    "mdma_tdt = 0 # Counter of samples where MDMA was used and tdTomato expressed\n",
    "meth_tdt = 0 # Counter of samples where Meth was used and tdTomato expressed\n",
    "\n",
    "sal_cfos = 0 # Counter of samples where Saline was used and cfos expressed\n",
    "mdma_cfos = 0 # Counter of samples where MDMA was used and cfos expressed\n",
    "meth_cfos = 0 # Counter of samples where Meth was used and cfos expressed\n",
    "\n",
    "#These array will contain sample's names as elements\n",
    "samples_sal_tdt = [] \n",
    "samples_mdma_tdt = [] \n",
    "samples_meth_tdt = [] \n",
    "samples_sal_cfos = []\n",
    "samples_mdma_cfos = [] \n",
    "samples_meth_cfos = []\n",
    "\n",
    "    ##############################################\n",
    "    # Processing each file\n",
    "    ##############################################\n",
    "\n",
    "while i < number_files:\n",
    "\n",
    "\n",
    "    file = list[i]\n",
    "    # Saving sample's number \n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "    # Reading file's data\n",
    "    header = [\"PIXEL_COUNT\"]\n",
    "    pixels_count_volume = pd.read_csv(file, names = header)\n",
    "    pixels_count_volume.insert(0, column = 'INTENSITY', value = range(0, len(pixels_count_volume))) \n",
    "\n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = intensities_template.merge(pixels_count_volume, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    volume_label_name = 'VOLUME_IN_TISSUE'\n",
    "    output = output.rename(columns = {'PIXEL_COUNT':volume_label_name})\n",
    "    output['VOLUME_IN_TISSUE'] = output['VOLUME_IN_TISSUE'].div(1000000).round(6)\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    else:\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "#     output['SAMPLE'] = sample\n",
    "    \n",
    "    # Merging activity and sample overview data-frames\n",
    "    output= pd.merge(output,samples_overview[['SAMPLE','GENDER',\n",
    "                                             'hemisphere','marker',\n",
    "                                             'TX_GROUP','CONTEXT']],on='SAMPLE',how='inner')\n",
    "    \n",
    "    #Adding a column with the sample's nickname\n",
    "\n",
    "    if (output['TX_GROUP'].iloc[0] == 'Saline') and (output['marker'].iloc[0] == 'tdtomato'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_sal_tdt:\n",
    "            sal_tdt += 1\n",
    "        samples_sal_tdt.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'vSAL_' + str(sal_tdt)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'MDMA') and (output['marker'].iloc[0] == 'tdtomato'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_mdma_tdt:\n",
    "            mdma_tdt += 1 \n",
    "        samples_mdma_tdt.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'vMDMA_' + str(mdma_tdt)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'METH') and (output['marker'].iloc[0] == 'tdtomato'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_meth_tdt:\n",
    "            meth_tdt += 1 \n",
    "        samples_meth_tdt.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'vMETH_' + str(meth_tdt)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'Saline') and (output['marker'].iloc[0] == 'cfos'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_sal_cfos:\n",
    "            sal_cfos += 1 \n",
    "        samples_sal_cfos.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'vSAL_' + str(sal_cfos)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'MDMA') and (output['marker'].iloc[0] == 'cfos'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_mdma_cfos:\n",
    "            mdma_cfos += 1 \n",
    "        samples_mdma_cfos.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'vMDMA_' + str(mdma_cfos)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'METH') and (output['marker'].iloc[0] == 'cfos'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_meth_cfos:\n",
    "            meth_cfos += 1 \n",
    "        samples_meth_cfos.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'vMETH_' + str(meth_cfos)\n",
    "\n",
    "    nickname_column = np.empty(len(output))    \n",
    "    output['nickname'] = nickname_column\n",
    "    output['nickname'] = nickname\n",
    "    \n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('L','',1)\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    volume_intissue_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframeo\n",
    "volume_intissue_data = pd.concat(volume_intissue_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculating volume ratio\n",
    "volumes_data = pd.merge(volume_intissue_data,volume_data[['raw','VOLUME','SAMPLE']],on=('raw','SAMPLE'),how='left')\n",
    "volumes_data['volumes_ratio'] = volumes_data['VOLUME_IN_TISSUE'] / volumes_data['VOLUME'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pivoting dataframe to have nicknames as columns\n",
    "pivoted_volumes_data = pd.pivot_table(volumes_data, values='VOLUME_IN_TISSUE', \n",
    "                                       index=['raw'],columns=['nickname'], aggfunc=np.sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"01Activity_files\" folder\n",
    "os.chdir(subfolder_with_path[0])\n",
    "\n",
    "#Counting number of files on \"01Activity_files\" folder\n",
    "list = os.listdir(subfolder_with_path[0]) \n",
    "number_files = len(list)\n",
    "number_activity_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "activity_data = [] #This array will contain df as elements\n",
    "\n",
    "#Staring counters\n",
    "\n",
    "i = 0 #General loop counter\n",
    "\n",
    "sal_tdt = 0 # Counter of samples where Saline was used and tdTomato expressed\n",
    "mdma_tdt = 0 # Counter of samples where MDMA was used and tdTomato expressed\n",
    "meth_tdt = 0 # Counter of samples where Meth was used and tdTomato expressed\n",
    "\n",
    "sal_cfos = 0 # Counter of samples where Saline was used and cfos expressed\n",
    "mdma_cfos = 0 # Counter of samples where MDMA was used and cfos expressed\n",
    "meth_cfos = 0 # Counter of samples where Meth was used and cfos expressed\n",
    "\n",
    "#These array will contain sample's names as elements\n",
    "samples_sal_tdt = [] \n",
    "samples_mdma_tdt = [] \n",
    "samples_meth_tdt = [] \n",
    "samples_sal_cfos = []\n",
    "samples_mdma_cfos = [] \n",
    "samples_meth_cfos = []\n",
    "\n",
    "\n",
    "    ##############################################\n",
    "    # Innitiates loop to process each file\n",
    "    ##############################################\n",
    "\n",
    "\n",
    "while i < number_files:\n",
    "\n",
    "    # Working with each file at a time\n",
    "    file = list[i]\n",
    "    \n",
    "    # Saving sample's number\n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "    # Reading file's data\n",
    "    regions_count_activity_fracc = pd.read_csv(file,low_memory=False)\n",
    "    regions_count_activity_fracc = regions_count_activity_fracc[regions_count_activity_fracc.PIXEL_COUNT != \"PIXEL_COUNT\"]        \n",
    "\n",
    "    # Adding-up regions' fracctions\n",
    "    regions_fracc_1 = regions_count_activity_fracc[[\"INTENSITY_1\", \"INTENSITY_1_PERC\"]]\n",
    "    regions_fracc_2 = regions_count_activity_fracc[[\"INTENSITY_2\", \"INTENSITY_2_PERC\"]]\n",
    "    regions_fracc_3 = regions_count_activity_fracc[[\"INTENSITY_3\", \"INTENSITY_3_PERC\"]]\n",
    "    regions_fracc_1 = regions_fracc_1.rename(columns={\"INTENSITY_1\": \"INTENSITY\",\"INTENSITY_1_PERC\" : \"COUNTS\"})\n",
    "    regions_fracc_2 = regions_fracc_2.rename(columns={\"INTENSITY_2\": \"INTENSITY\",\"INTENSITY_2_PERC\" : \"COUNTS\"})\n",
    "    regions_fracc_3 = regions_fracc_3.rename(columns={\"INTENSITY_3\": \"INTENSITY\",\"INTENSITY_3_PERC\" : \"COUNTS\"})\n",
    "    total_region_activity = regions_fracc_1.append(regions_fracc_2, \n",
    "                                                   ignore_index=True).append(regions_fracc_3, ignore_index=True)\n",
    "    total_region_activity[\"COUNTS\"] = total_region_activity[\"COUNTS\"].astype(float)\n",
    "    total_region_activity = total_region_activity.groupby(['INTENSITY']).agg('sum').reset_index()\n",
    "#     total_region_activity[\"COUNTS\"] = total_region_activity[\"COUNTS\"].round(0)\n",
    "    total_region_activity[\"INTENSITY\"] = total_region_activity[\"INTENSITY\"].astype(int)\n",
    "        \n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = intensities_template.merge(total_region_activity, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    activity_label_name = 'ACTIVITY'\n",
    "    output = output.rename(columns = {'COUNTS':activity_label_name})\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    else:\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "\n",
    "    \n",
    "#     # Adding a column of the rater's name\n",
    "#     rater_column = np.empty(len(output))\n",
    "#     output['rater'] = rater_column\n",
    "#     output['rater'] = rater\n",
    "    \n",
    "    # Merging activity and sample overview data-frames\n",
    "    output= pd.merge(output,samples_overview[['SAMPLE','GENDER',\n",
    "                                             'hemisphere','marker',\n",
    "                                             'TX_GROUP','CONTEXT']],on='SAMPLE',how='inner')\n",
    "    \n",
    "    \n",
    "    #Adding a column with the sample's nickname\n",
    "\n",
    "    if (output['TX_GROUP'].iloc[0] == 'Saline') and (output['marker'].iloc[0] == 'tdtomato'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_sal_tdt:\n",
    "            sal_tdt += 1\n",
    "        samples_sal_tdt.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'SAL_' + str(sal_tdt)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'MDMA') and (output['marker'].iloc[0] == 'tdtomato'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_mdma_tdt:\n",
    "            mdma_tdt += 1 \n",
    "        samples_mdma_tdt.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'MDMA_' + str(mdma_tdt)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'METH') and (output['marker'].iloc[0] == 'tdtomato'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_meth_tdt:\n",
    "            meth_tdt += 1 \n",
    "        samples_meth_tdt.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'METH_' + str(meth_tdt)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'Saline') and (output['marker'].iloc[0] == 'cfos'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_sal_cfos:\n",
    "            sal_cfos += 1 \n",
    "        samples_sal_cfos.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'SAL_' + str(sal_cfos)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'MDMA') and (output['marker'].iloc[0] == 'cfos'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_mdma_cfos:\n",
    "            mdma_cfos += 1 \n",
    "        samples_mdma_cfos.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'MDMA_' + str(mdma_cfos)\n",
    "        \n",
    "    elif (output['TX_GROUP'].iloc[0] == 'METH') and (output['marker'].iloc[0] == 'cfos'):\n",
    "        if output['SAMPLE'].iloc[0] not in samples_meth_cfos:\n",
    "            meth_cfos += 1 \n",
    "        samples_meth_cfos.append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'METH_' + str(meth_cfos)\n",
    "\n",
    "    nickname_column = np.empty(len(output))    \n",
    "    output['nickname'] = nickname_column\n",
    "    output['nickname'] = nickname\n",
    "\n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('L','',1)\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    activity_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframe\n",
    "activity_data = pd.concat(activity_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Replacing empty cells with zeros\n",
    "activity_data['ACTIVITY'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from decimal import localcontext, Decimal, ROUND_HALF_UP\n",
    "# with localcontext() as ctx: ctx.rounding = ROUND_HALF_UP\n",
    "# data = {'values': [5.52132, 6.572935, 7.21, 8.755, 9.9989]}\n",
    "# df = pd.DataFrame(data, columns = ['values'])\n",
    "# # df['values'].apply(np.ceil)\n",
    "# Decimal(df['values'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Merging activity and volume ratio to filter activity where the volume ratio is below 50%\n",
    "filtered_activity_data = pd.merge(activity_data,\n",
    "                                  volumes_data[['raw','SAMPLE','volumes_ratio']],\n",
    "                                  on=['raw','SAMPLE'],\n",
    "                                  how='left')\n",
    "filtered_activity_data.loc[(filtered_activity_data['volumes_ratio'] < 0.5) | (filtered_activity_data['volumes_ratio'] > 1.0)\n",
    "                           ,'ACTIVITY'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pivoting dataframe to have nicknames as columns\n",
    "pivoted_activity_data = pd.pivot_table(filtered_activity_data, values='ACTIVITY', \n",
    "                               index=['raw','hemisphere','marker'],columns=['nickname'], aggfunc=np.sum)\n",
    "\n",
    "#Restarting index to activate'hemisphere','marker'and 'rater' as columns\n",
    "pivoted_activity_data = pivoted_activity_data.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the main path\n",
    "os.chdir(main_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combining dataframes to create the \"cell_count_output.csv\" file\n",
    "allen_and_activity = pd.merge(brain_allen_map,pivoted_activity_data,on='raw',how='right')\n",
    "cell_count_output = pd.merge(allen_and_activity,pivoted_volumes_data,on='raw',how='left')\n",
    "cell_count_output.to_csv('ALLEN_cell_count_output.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, ' Files processed ')\n",
      " \n",
      "(' Execution time:', 2.36, 'seconds')\n"
     ]
    }
   ],
   "source": [
    "#Ending timming process\n",
    "end = time.time()\n",
    "\n",
    "# Printing how many files were processed and how much time the process took\n",
    "files_processed = number_activity_files = number_files+ number_volume_files + number_volintissue_files\n",
    "print(files_processed,' Files processed ')\n",
    "print(' ')\n",
    "print(' Execution time:', round((end - start),2), 'seconds') "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}