workflows/analysis/compute_best_param.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "82a3402c-5861-4040-a826-c5cfaaecf1ce",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### This notebook computes the median ARI score across different hyperparameter settings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "870f6a36-25d6-41c0-9a5a-e42a574c78fb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "#os.chdir('../../10x_tupro/')\n",
    "#os.chdir('../../maynard_human_brain_analysis/')\n",
    "#os.chdir('../../her2_positive_breast_tumors/')\n",
    "#os.chdir('../../../../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bc93f20-6daa-4069-8fdf-b6d816b137b9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.cluster import adjusted_rand_score\n",
    "from tqdm import tqdm\n",
    "import scanpy as sc\n",
    "import pandas as pd\n",
    "import shutil\n",
    "import glob\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7511d966-0e9c-4615-bc96-5a4c5a0ce046",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from time import gmtime, strftime\n",
    "strftime(\"%Y-%m-%d %H:%M:%S\", gmtime())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec273661-6df5-4cab-8a56-34380ac907ba",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "model = \"AESTETIK\"\n",
    "out_folder = \"out_benchmark\"\n",
    "cross_validation_combination = \"151675_151676_test_151507_151508_151509_151510_151669_151670_151671_151672_151673_151674\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06106d66-cfb2-4881-9a71-9f59e65cf727",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "samples = cross_validation_combination.split(\"_test\")[0].replace(\"split_train_\", \"\").split(\"_\")\n",
    "samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d5ad865-fb1c-42b8-be3e-f9ce4e5796d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ari_result = []\n",
    "for sample in tqdm(samples):\n",
    "    # Load adata\n",
    "    adata_path = os.path.join(out_folder, \"data\", \"h5ad\", f\"{sample}.h5ad\")\n",
    "    adata = sc.read(adata_path)\n",
    "    adata.obs[\"Barcode\"] = adata.obs.index.values\n",
    "    \n",
    "    # Find cluster paths\n",
    "    cluster_paths = glob.glob(os.path.join(out_folder, cross_validation_combination, f\"{model}_fine_tune\", \"clusters\", f\"model-{sample}-*.csv\"))\n",
    "    \n",
    "    for cluster_path in cluster_paths:\n",
    "        # Extract parameter name from the cluster path\n",
    "        param_name = os.path.splitext(os.path.basename(cluster_path))[0].replace(f\"model-{sample}-\", \"\")\n",
    "        \n",
    "        # Read cluster data\n",
    "        df = pd.read_csv(cluster_path)\n",
    "        cluster_label_dict = pd.Series(df[df.columns[1]].values, index=df[df.columns[0]].values).to_dict()\n",
    "        \n",
    "        # Assign cluster labels to adata\n",
    "        adata.obs[param_name] = adata.obs.Barcode.map(cluster_label_dict).astype(str)\n",
    "        \n",
    "        # Calculate ARI\n",
    "        ari = adjusted_rand_score(adata.obs.ground_truth, adata.obs[param_name])\n",
    "        \n",
    "        # Append results\n",
    "        ari_result.append([param_name, sample, ari])\n",
    "\n",
    "# Create DataFrame\n",
    "ari_result = pd.DataFrame(ari_result, columns=[\"model\", \"sample\", \"ari\"])\n",
    "ari_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a048791-3c23-4070-b744-edb93f23159b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# groupby param and compute median, sorted by ARI\n",
    "ari_result = ari_result.groupby(\"model\").ari.agg(\"median\").reset_index().sort_values(\"ari\", ascending=False)\n",
    "ari_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4f3fef8-cf18-431f-b142-a9fdf5f47ce3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_model = ari_result.model.values[0]\n",
    "top_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b63e95-01f5-4429-8262-c078d11e2d7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameter_path = f\"{out_folder}/{cross_validation_combination}/{model}_fine_tune/parameters/{top_model}.yaml\"\n",
    "parameter_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e75e73e-75f5-4aff-a193-d509f77477c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# my best parameters \n",
    "with open(parameter_path, \"r\") as stream:\n",
    "    parameters = yaml.safe_load(stream)\n",
    "parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69b86e73-a0cb-4a89-9e44-463986443d03",
   "metadata": {},
   "outputs": [],
   "source": [
    "shutil.copy(parameter_path, f\"{out_folder}/{cross_validation_combination}/{model}_fine_tune/parameters/best_param.yaml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d71876b8-f78a-4167-87e1-81d88d6ae95a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nonchev",
   "language": "python",
   "name": "nonchev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}