DS-Data-Catalog.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6474f65c-a0da-4e64-9251-439f77d8d5a7",
   "metadata": {},
   "source": [
    "# Data Catalog of DS-Students-Resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cc3f3fd0-a124-4a42-9e78-df00fe2640af",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Import the necessary libraries \n",
    "import os\n",
    "import pandas as pd\n",
    "import glob\n",
    "from pathlib import Path\n",
    "import csv\n",
    "from datetime import datetime\n",
    "import mimetypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dd01b26b-c343-4fec-9354-9f850e326c82",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# function to recursively find all dataset files in local GitHub repository ignoring .ipynb_checkpoint files\n",
    "def find_datasets(root_path, extensions):\n",
    "    datasets = []\n",
    "    for ext in extensions:\n",
    "        datasets.extend([path for path in Path(root_path).rglob(f\"*.{ext}\") if \".ipynb_checkpoints\" not in str(path)])\n",
    "    return datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0c37eb7-1553-4d03-93ee-aced669b5715",
   "metadata": {},
   "source": [
    "### List (not exhaustive) of dataset file formats and their corresponding extensions:\n",
    "\n",
    "- CSV (Comma Separated Values): csv\n",
    "- TSV (Tab Separated Values): tsv\n",
    "- JSON (JavaScript Object Notation): json\n",
    "- Excel: xls, xlsx\n",
    "- Plain Text: txt\n",
    "- Data: data\n",
    "- HDF5 (Hierarchical Data Format): h5, hdf5\n",
    "- NetCDF (Network Common Data Form): nc, nc4\n",
    "- XML (eXtensible Markup Language): xml\n",
    "- Parquet: parquet\n",
    "- Avro: avro\n",
    "- Feather: feather\n",
    "- ORC (Optimized Row Columnar): orc\n",
    "- Protocol Buffers: pb, pbf\n",
    "- GeoJSON: geojson\n",
    "- Pickle: pkl, pickle\n",
    "- MATLAB: mat\n",
    "- ARFF (Attribute-Relation File Format): arff\n",
    "- NPY (NumPy array): npy\n",
    "- NPZ (NumPy Zipped): npz\n",
    "- SAS: sas7bdat\n",
    "- STATA: dta\n",
    "- R (RData, RDS): RData, rds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eb09a50a-ce18-4530-ade4-358f5b2595ed",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define the extensions of dataset files you want to search for and call the find_datasets function.\n",
    "dataset_extensions = [\n",
    "    \"csv\", \"tsv\", \"json\", \"xlsx\", \"xls\", \"txt\", \"data\", \"h5\", \"hdf5\",\n",
    "    \"nc\", \"nc4\", \"xml\", \"parquet\", \"avro\", \"feather\", \"orc\", \"pb\", \"pbf\",\n",
    "    \"geojson\", \"pkl\", \"pickle\", \"mat\", \"arff\", \"npy\", \"npz\", \"sas7bdat\",\n",
    "    \"dta\", \"RData\", \"rds\"\n",
    "]\n",
    "\n",
    "root_dir = \".\"  # The root directory of local GitHub repository\n",
    "datasets = find_datasets(root_dir, dataset_extensions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d039f312-1a7b-4adc-9354-2bb6375c2801",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create a function to extract the necessary metadata from each dataset file.\n",
    "def get_dataset_metadata(dataset_path):\n",
    "    metadata = {\n",
    "        \"filename\": dataset_path.name,\n",
    "        \"path\": str(dataset_path),\n",
    "        \"extension\": dataset_path.suffix[1:],\n",
    "        \"size\": dataset_path.stat().st_size,\n",
    "        \"mime_type\": mimetypes.guess_type(dataset_path)[0] or \"unknown\",\n",
    "        \"modified_date\": datetime.fromtimestamp(dataset_path.stat().st_mtime).strftime('%Y-%m-%d %H:%M:%S'),\n",
    "    }\n",
    "    return metadata\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e86e374a-ece4-43ec-bb61-e442f1fc3f25",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create a Pandas DataFrame to store the dataset metadata and write it to a CSV file.\n",
    "dataset_metadata = [get_dataset_metadata(ds) for ds in datasets]\n",
    "metadata_df = pd.DataFrame(dataset_metadata)\n",
    "metadata_df.to_csv(\"ds-datasets.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ba5fcde9-4063-482c-91f3-6c65e88c8d8d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>path</th>\n",
       "      <th>extension</th>\n",
       "      <th>size</th>\n",
       "      <th>mime_type</th>\n",
       "      <th>modified_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ds-video-catalog.csv</td>\n",
       "      <td>ds-video-catalog.csv</td>\n",
       "      <td>csv</td>\n",
       "      <td>58906</td>\n",
       "      <td>text/csv</td>\n",
       "      <td>2023-05-15 18:01:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ds-datasets.csv</td>\n",
       "      <td>ds-datasets.csv</td>\n",
       "      <td>csv</td>\n",
       "      <td>75844</td>\n",
       "      <td>text/csv</td>\n",
       "      <td>2023-05-27 07:30:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ds-jupyter-notebooks.csv</td>\n",
       "      <td>ds-jupyter-notebooks.csv</td>\n",
       "      <td>csv</td>\n",
       "      <td>99403</td>\n",
       "      <td>text/csv</td>\n",
       "      <td>2023-05-27 07:31:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DS-Students.csv</td>\n",
       "      <td>DS108-Databases/NoSQL/Data/DS-Students.csv</td>\n",
       "      <td>csv</td>\n",
       "      <td>12452</td>\n",
       "      <td>text/csv</td>\n",
       "      <td>2021-11-18 13:17:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>addresses.csv</td>\n",
       "      <td>DS108-Databases/SQL/Workshops/Week-2/08-Export...</td>\n",
       "      <td>csv</td>\n",
       "      <td>52009</td>\n",
       "      <td>text/csv</td>\n",
       "      <td>2021-12-03 16:18:53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   filename   \n",
       "0      ds-video-catalog.csv  \\\n",
       "1           ds-datasets.csv   \n",
       "2  ds-jupyter-notebooks.csv   \n",
       "3           DS-Students.csv   \n",
       "4             addresses.csv   \n",
       "\n",
       "                                                path extension   size   \n",
       "0                               ds-video-catalog.csv       csv  58906  \\\n",
       "1                                    ds-datasets.csv       csv  75844   \n",
       "2                           ds-jupyter-notebooks.csv       csv  99403   \n",
       "3         DS108-Databases/NoSQL/Data/DS-Students.csv       csv  12452   \n",
       "4  DS108-Databases/SQL/Workshops/Week-2/08-Export...       csv  52009   \n",
       "\n",
       "  mime_type        modified_date  \n",
       "0  text/csv  2023-05-15 18:01:33  \n",
       "1  text/csv  2023-05-27 07:30:58  \n",
       "2  text/csv  2023-05-27 07:31:14  \n",
       "3  text/csv  2021-11-18 13:17:16  \n",
       "4  text/csv  2021-12-03 16:18:53  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list the metadata contained in the metadata_df\n",
    "metadata_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ecdbe98d-1b1c-4951-9cf4-530cf0d55345",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total size of all Datasets: 919.12 MB\n"
     ]
    }
   ],
   "source": [
    "# computer and print total size of all datasets\n",
    "total_size_bytes = metadata_df[\"size\"].sum()\n",
    "total_size_mb = total_size_bytes / (1024 ** 2)  # Convert bytes to megabytes\n",
    "print(f\"Total size of all Datasets: {total_size_mb:.2f} MB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "10c3d906-e3f7-412b-86d9-4c5f0815f8cc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of Datasets: 566\n"
     ]
    }
   ],
   "source": [
    "# count total number of datasets\n",
    "total_datasets = len(datasets)\n",
    "print(f\"Total number of Datasets: {total_datasets}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c3b8ec5-55ad-49b3-9c4d-9e98342666fd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}