cell_counts_and_region_volumes_generic_092021.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing all libraries needed\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stanting timming process\n",
    "start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the main directory\n",
    "main_directory = os.getcwd()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rreading the name of the subfolder\n",
    "# subfolder_with_path = [f.path for f in os.scandir(main_directory) if f.is_dir()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subfolder_name = [f.name for f in os.scandir(main_directory) if f.is_dir()]\n",
    "subfolder_with_path = [\"\" for i in range(4)]\n",
    "subfolder_with_path[0] = main_directory + '/'+'01Activity_files'\n",
    "subfolder_with_path[1] = main_directory + '/'+'02Volume_files'\n",
    "subfolder_with_path[2] = main_directory + '/'+'03Volume_in_tissue_files'\n",
    "subfolder_with_path[3] = main_directory + '/'+'zzz_input_and_reference_files'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################################\n",
    "# Importing information from \"samples overview\", \"intensisties template\" and \"brain allen map\"\n",
    "###############################################################################################\n",
    "\n",
    "# Changing directory to the \"zzz_input_and_reference_files\" folder\n",
    "os.chdir(subfolder_with_path[-1])\n",
    "\n",
    "\n",
    "# Reading \"sample_overview.csv\" file\n",
    "samples_overview = pd.read_csv(\"sample_overview.csv\", \n",
    "                               dtype={\"MOUSE\": \"string\",\"SAMPLE\": int,\n",
    "                                      \"FILES_HD\": int,\"FILES_BOX\": int,\n",
    "                                      \"hemisphere\": \"string\",\"MARKER\": \"string\",\n",
    "                                      \"TX_GROUP\": \"string\", \n",
    "                                      \"GENDER\":\"string\", \n",
    "                                      \"CONTEXT\":\"string\"})\n",
    "\n",
    "# Reading \"intensities_template.csv\" file\n",
    "intensities_template = pd.read_csv(\"intensities_template.csv\",\n",
    "                                   dtype={\"IDPath\": \"string\", \n",
    "                                          \"LabelID\": int, \n",
    "                                          \"raw\": \"string\", \n",
    "                                          \"labelabrv\": \"string\"})\n",
    "# Reading \"brain_allen_map.csv\" file\n",
    "brain_allen_map = pd.read_csv(\"brain_allen_map.csv\", \n",
    "                               dtype={\"raw\": \"string\",\"allen_1\": \"string\",\n",
    "                                      \"allen_2\": \"string\",\"allen_3\": \"string\",\n",
    "                                      \"allen_4\": \"string\",\"allen_5\": \"string\",\n",
    "                                      \"allen_6\": \"string\",\"allen_7\": \"string\",\n",
    "                                      \"fine\": \"string\",\"medium\": \"string\",\n",
    "                                      \"coarse\": \"string\", \"all\":\"string\"})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving drugs and markers' names from the \"sample_overview\" file\n",
    "\n",
    "drugs = samples_overview['TX_GROUP'].unique()\n",
    "markers = samples_overview['marker'].unique()\n",
    "\n",
    "d = 0\n",
    "m = 0\n",
    "counters_names = []\n",
    "while (m < len(markers)):\n",
    "    while (d < len(drugs)):\n",
    "        counters_names.append(drugs[d] + '_' + markers[m])\n",
    "        d += 1\n",
    "    m += 1\n",
    "    d = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"02Volume_files\" folder\n",
    "os.chdir(subfolder_with_path[1])\n",
    "\n",
    "#Counting number of files on \"02Volume_files\" folder\n",
    "list = os.listdir(subfolder_with_path[1]) \n",
    "number_files = len(list)\n",
    "number_volume_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "volume_data = [] #This array will contain df as elements\n",
    "\n",
    "i = 0\n",
    "while i < number_files:\n",
    "\n",
    "    ##############################################\n",
    "    # Processing each file\n",
    "    ##############################################\n",
    "    \n",
    "    file = list[i]\n",
    "    # Saving sample's number\n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "\n",
    "    # Reading file's data\n",
    "    header = [\"PIXEL_COUNT\"]\n",
    "    pixels_count_volume = pd.read_csv(file, names = header)\n",
    "    pixels_count_volume.insert(0, column = 'INTENSITY', value = range(0, len(pixels_count_volume))) \n",
    "\n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = intensities_template.merge(pixels_count_volume, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    volume_label_name = 'VOLUME'\n",
    "    output = output.rename(columns = {'PIXEL_COUNT':volume_label_name})\n",
    "    output['VOLUME'] = output['VOLUME'].div(1000000).round(7)\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    else:\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "   \n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "    \n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('L','',1)\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    volume_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframe\n",
    "volume_data = pd.concat(volume_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"03Volume_in_tissue_files\" folder\n",
    "os.chdir(subfolder_with_path[2])\n",
    "\n",
    "#Counting number of files on \"03Volume_in_tissue_files\" folder\n",
    "list = os.listdir(subfolder_with_path[2]) \n",
    "number_files = len(list)\n",
    "number_volintissue_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "volume_intissue_data = [] #This array will contain df as elements\n",
    "\n",
    "#Staring counters\n",
    "\n",
    "i = 0 #General loop counter\n",
    "\n",
    "\n",
    "#Zeroing the counters \n",
    "counters = np.zeros(len(counters_names))\n",
    "\n",
    "#Creating counters array to contain sample's names as elements\n",
    "counters_samples = [[] for i in range(len(counters_names))]\n",
    "\n",
    "    ##############################################\n",
    "    # Processing each file\n",
    "    ##############################################\n",
    "\n",
    "while i < number_files:\n",
    "\n",
    "\n",
    "    file = list[i]\n",
    "    # Saving sample's number \n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "    # Reading file's data\n",
    "    header = [\"PIXEL_COUNT\"]\n",
    "    pixels_count_volume = pd.read_csv(file, names = header)\n",
    "    pixels_count_volume.insert(0, column = 'INTENSITY', value = range(0, len(pixels_count_volume))) \n",
    "\n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = intensities_template.merge(pixels_count_volume, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    volume_label_name = 'VOLUME_IN_TISSUE'\n",
    "    output = output.rename(columns = {'PIXEL_COUNT':volume_label_name})\n",
    "    output['VOLUME_IN_TISSUE'] = output['VOLUME_IN_TISSUE'].div(1000000).round(6)\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    else:\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "    \n",
    "    # Merging activity and sample overview data-frames\n",
    "    output= pd.merge(output,samples_overview[['SAMPLE','GENDER',\n",
    "                                             'hemisphere','marker',\n",
    "                                             'TX_GROUP','CONTEXT']],on='SAMPLE',how='inner')\n",
    "    \n",
    "    #Adding a column with the sample's nickname\n",
    "\n",
    "    counters_index = counters_names.index(output['TX_GROUP'].iloc[0] + '_' + output['marker'].iloc[0])\n",
    "    if output['SAMPLE'].iloc[0] not in counters_samples[counters_index]:\n",
    "        counters[counters_index] += 1\n",
    "        counters_samples[counters_index].append(output['SAMPLE'].iloc[0])\n",
    "        nickname = 'v' + output['TX_GROUP'].iloc[0] + '_' + str(int(counters[counters_index]))\n",
    "\n",
    "    nickname_column = np.empty(len(output))    \n",
    "    output['nickname'] = nickname_column\n",
    "    output['nickname'] = nickname\n",
    "    \n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('L','',1)\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    volume_intissue_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframeo\n",
    "volume_intissue_data = pd.concat(volume_intissue_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculating volume ratio\n",
    "volumes_data = pd.merge(volume_intissue_data,volume_data[['raw','VOLUME','SAMPLE']],on=('raw','SAMPLE'),how='left')\n",
    "volumes_data['volumes_ratio'] = volumes_data['VOLUME_IN_TISSUE'] / volumes_data['VOLUME'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pivoting dataframe to have nicknames as columns\n",
    "pivoted_volumes_data = pd.pivot_table(volumes_data, values='VOLUME_IN_TISSUE', \n",
    "                                       index=['raw'],columns=['nickname'], aggfunc=np.sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the \"01Activity_files\" folder\n",
    "os.chdir(subfolder_with_path[0])\n",
    "\n",
    "#Counting number of files on \"01Activity_files\" folder\n",
    "list = os.listdir(subfolder_with_path[0]) \n",
    "number_files = len(list)\n",
    "number_activity_files = number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using and timing  a loop to process all files and populate the main data-frame\n",
    "\n",
    "activity_data = [] #This array will contain df as elements\n",
    "\n",
    "#Staring counters\n",
    "\n",
    "i = 0 #General loop counter\n",
    "\n",
    "#Zeroing the counters \n",
    "counters = np.zeros(len(counters_names))\n",
    "\n",
    "#Creating counters array to contain sample's names as elements\n",
    "counters_samples = [[] for i in range(len(counters_names))]\n",
    "\n",
    "\n",
    "    ##############################################\n",
    "    # Innitiates loop to process each file\n",
    "    ##############################################\n",
    "\n",
    "\n",
    "while i < number_files:\n",
    "\n",
    "    # Working with each file at a time\n",
    "    file = list[i]\n",
    "    \n",
    "    # Saving sample's number\n",
    "    filename_segmented = file.split('_')\n",
    "    sample = filename_segmented[1]\n",
    "\n",
    "    # Reading file's data\n",
    "    regions_count_activity_fracc = pd.read_csv(file,low_memory=False)\n",
    "    regions_count_activity_fracc = regions_count_activity_fracc[regions_count_activity_fracc.PIXEL_COUNT != \"PIXEL_COUNT\"]        \n",
    "\n",
    "    # Adding-up regions' fracctions\n",
    "    regions_fracc_1 = regions_count_activity_fracc[[\"INTENSITY_1\", \"INTENSITY_1_PERC\"]]\n",
    "    regions_fracc_2 = regions_count_activity_fracc[[\"INTENSITY_2\", \"INTENSITY_2_PERC\"]]\n",
    "    regions_fracc_3 = regions_count_activity_fracc[[\"INTENSITY_3\", \"INTENSITY_3_PERC\"]]\n",
    "    regions_fracc_1 = regions_fracc_1.rename(columns={\"INTENSITY_1\": \"INTENSITY\",\"INTENSITY_1_PERC\" : \"COUNTS\"})\n",
    "    regions_fracc_2 = regions_fracc_2.rename(columns={\"INTENSITY_2\": \"INTENSITY\",\"INTENSITY_2_PERC\" : \"COUNTS\"})\n",
    "    regions_fracc_3 = regions_fracc_3.rename(columns={\"INTENSITY_3\": \"INTENSITY\",\"INTENSITY_3_PERC\" : \"COUNTS\"})\n",
    "    total_region_activity = regions_fracc_1.append(regions_fracc_2, \n",
    "                                                   ignore_index=True).append(regions_fracc_3, ignore_index=True)\n",
    "    total_region_activity[\"COUNTS\"] = total_region_activity[\"COUNTS\"].astype(float)\n",
    "    total_region_activity = total_region_activity.groupby(['INTENSITY']).agg('sum').reset_index()\n",
    "#     total_region_activity[\"COUNTS\"] = total_region_activity[\"COUNTS\"].round(0)\n",
    "    total_region_activity[\"INTENSITY\"] = total_region_activity[\"INTENSITY\"].astype(int)\n",
    "        \n",
    "    # Adding activity columns to intensitites template data-frame\n",
    "    output = intensities_template.merge(total_region_activity, how='left', left_on='LabelID', right_on='INTENSITY')\n",
    "    output = output.drop(columns=['INTENSITY'])\n",
    "    activity_label_name = 'ACTIVITY'\n",
    "    output = output.rename(columns = {'COUNTS':activity_label_name})\n",
    "    \n",
    "    # Checking to which hemisphere the sample belongs to and erasing activity that belong to the other hemisphere\n",
    "    hemisphere = samples_overview['hemisphere'].loc[samples_overview['SAMPLE'] == int(sample)]\n",
    "    hemisphere = hemisphere.array\n",
    "    if hemisphere[0] == 'left':\n",
    "        output = output[output['LabelID'] > 20000]\n",
    "    else:\n",
    "        output = output[output['LabelID'] < 20000]\n",
    "\n",
    "    # Adding a column of the sample number\n",
    "    sample_column = np.empty(len(output))\n",
    "    sample = int(sample)\n",
    "    sample_column.fill(sample)\n",
    "    output['SAMPLE'] = sample_column\n",
    "    \n",
    "    # Merging activity and sample overview data-frames\n",
    "    output= pd.merge(output,samples_overview[['SAMPLE','GENDER',\n",
    "                                             'hemisphere','marker',\n",
    "                                             'TX_GROUP','CONTEXT']],on='SAMPLE',how='inner')\n",
    "    \n",
    "    \n",
    "    #Adding a column with the sample's nickname\n",
    "\n",
    "    counters_index = counters_names.index(output['TX_GROUP'].iloc[0] + '_' + output['marker'].iloc[0])\n",
    "    if output['SAMPLE'].iloc[0] not in counters_samples[counters_index]:\n",
    "        counters[counters_index] += 1\n",
    "        counters_samples[counters_index].append(output['SAMPLE'].iloc[0])\n",
    "        nickname = output['TX_GROUP'].iloc[0] + '_' + str(int(counters[counters_index]))\n",
    "\n",
    "    nickname_column = np.empty(len(output))    \n",
    "    output['nickname'] = nickname_column\n",
    "    output['nickname'] = nickname\n",
    "\n",
    "    # Erasing the hemisphere label form the \"raw\" and the and \"labelabrv\" columns\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('left_','')\n",
    "    output[\"raw\"] = output[\"raw\"].str.replace('right_','')\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('L','',1)\n",
    "    output[\"labelabrv\"] = output[\"labelabrv\"].str.replace('R','',1)\n",
    "    \n",
    "    ##########################################################################################################################\n",
    "    \n",
    "    # store DataFrame in list\n",
    "    activity_data.append(output)\n",
    "    \n",
    "    i += 1\n",
    "    \n",
    "#Joinning all elements of the array in a dataframe\n",
    "activity_data = pd.concat(activity_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Replacing empty cells with zeros\n",
    "activity_data['ACTIVITY'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating density dataframe\n",
    "##Merging activity, volume and volume in tissue dataframes\n",
    "density = pd.merge(activity_data,\n",
    "                                  volumes_data[['raw','SAMPLE','VOLUME']],\n",
    "                                  on=['raw','SAMPLE'],\n",
    "                                  how='left')\n",
    "density = pd.merge(density,\n",
    "                           volume_intissue_data[['raw','SAMPLE','VOLUME_IN_TISSUE']],\n",
    "                           on=['raw','SAMPLE'],\n",
    "                           how='left')\n",
    "#Creating the density nickname (d_nickname)\n",
    "density['DENSITY'] = density['ACTIVITY'] / density['VOLUME_IN_TISSUE']\n",
    "density['d_nickname'] = 'd_' + density['nickname']\n",
    "\n",
    "\n",
    "#Pivoting dataframe to have d_nicknames as columns\n",
    "pivoted_density = pd.pivot_table(density, values='DENSITY', \n",
    "                                       index=['raw'],columns=['d_nickname'], aggfunc=np.sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Merging activity and volume ratio to filter activity where the volume ratio is below 50%\n",
    "filtered_activity_data = pd.merge(activity_data,\n",
    "                                  volumes_data[['raw','SAMPLE','volumes_ratio']],\n",
    "                                  on=['raw','SAMPLE'],\n",
    "                                  how='left')\n",
    "filtered_activity_data.loc[(filtered_activity_data['volumes_ratio'] < 0.5),'ACTIVITY'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pivoting dataframe to have nicknames as columns\n",
    "pivoted_activity_data = pd.pivot_table(filtered_activity_data, values='ACTIVITY', \n",
    "                               index=['raw','hemisphere','marker'],columns=['nickname'], aggfunc=np.sum)\n",
    "\n",
    "#Restarting index to activate'hemisphere','marker'and 'rater' as columns\n",
    "pivoted_activity_data = pivoted_activity_data.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combining all dataframes\n",
    "allen_and_activity = pd.merge(brain_allen_map,pivoted_activity_data,on='raw',how='right')\n",
    "cell_count_output = pd.merge(allen_and_activity,pivoted_volumes_data,on='raw',how='left')\n",
    "cell_count_output = pd.merge(cell_count_output,pivoted_density,on='raw',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing directory to the main path\n",
    "os.chdir(main_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating the \"cell_count_output.csv\" file\n",
    "cell_count_output.to_csv('cell_count_output.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Ending timming process\n",
    "end = time.time()\n",
    "\n",
    "# Printing how many files were processed and how much time the process took\n",
    "files_processed = number_activity_files = number_files+ number_volume_files + number_volintissue_files\n",
    "print(files_processed,' Files processed ')\n",
    "print(' ')\n",
    "print(' Execution time:', round((end - start),2), 'seconds') "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}