guides/logit_checking/k400-logit-matching-torch-vs-keras-cv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cc7788a6",
   "metadata": {
    "papermill": {
     "duration": 0.006608,
     "end_time": "2024-03-28T18:55:53.448019",
     "exception": false,
     "start_time": "2024-03-28T18:55:53.441411",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# About\n",
    "\n",
    "This notebook demonstrates the identical results of vidoe swin transformer, imported from `keras-cv` and `torch-vision` libraries. The `keras-cv` version of video swin is implemented in `keras 3`, makes it able to run in multiple backend, i.e. `tensorflow`, `torch`, and `jax`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "86437e84",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-03-28T18:55:53.461887Z",
     "iopub.status.busy": "2024-03-28T18:55:53.461497Z",
     "iopub.status.idle": "2024-03-28T18:55:54.343326Z",
     "shell.execute_reply": "2024-03-28T18:55:54.342152Z"
    },
    "papermill": {
     "duration": 0.89173,
     "end_time": "2024-03-28T18:55:54.346095",
     "exception": false,
     "start_time": "2024-03-28T18:55:53.454365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6c8b9d13",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:55:54.359527Z",
     "iopub.status.busy": "2024-03-28T18:55:54.359011Z",
     "iopub.status.idle": "2024-03-28T18:55:54.364144Z",
     "shell.execute_reply": "2024-03-28T18:55:54.363316Z"
    },
    "papermill": {
     "duration": 0.014217,
     "end_time": "2024-03-28T18:55:54.366189",
     "exception": false,
     "start_time": "2024-03-28T18:55:54.351972",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.environ[\"KERAS_BACKEND\"] = \"torch\" # 'torch', 'tensorflow', 'jax'\n",
    "\n",
    "warnings.simplefilter(action=\"ignore\")\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9641510f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:55:54.379961Z",
     "iopub.status.busy": "2024-03-28T18:55:54.378975Z",
     "iopub.status.idle": "2024-03-28T18:56:26.782756Z",
     "shell.execute_reply": "2024-03-28T18:56:26.781252Z"
    },
    "papermill": {
     "duration": 32.413608,
     "end_time": "2024-03-28T18:56:26.785582",
     "exception": false,
     "start_time": "2024-03-28T18:55:54.371974",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'keras-cv'...\r\n",
      "remote: Enumerating objects: 13735, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (1872/1872), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (752/752), done.\u001b[K\r\n",
      "remote: Total 13735 (delta 1297), reused 1587 (delta 1104), pack-reused 11863\u001b[K\r\n",
      "Receiving objects: 100% (13735/13735), 25.64 MiB | 31.71 MiB/s, done.\r\n",
      "Resolving deltas: 100% (9742/9742), done.\r\n",
      "/kaggle/working/keras-cv\n"
     ]
    }
   ],
   "source": [
    "!git clone --branch video_swin https://github.com/innat/keras-cv.git\n",
    "%cd keras-cv\n",
    "!pip install -q -e ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbbc08fc",
   "metadata": {
    "papermill": {
     "duration": 0.007,
     "end_time": "2024-03-28T18:56:26.800170",
     "exception": false,
     "start_time": "2024-03-28T18:56:26.793170",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# KerasCV: Video Swin : Pretrained: ImageNet 1K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a42e9c56",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:56:26.816883Z",
     "iopub.status.busy": "2024-03-28T18:56:26.816366Z",
     "iopub.status.idle": "2024-03-28T18:56:52.607621Z",
     "shell.execute_reply": "2024-03-28T18:56:52.606236Z"
    },
    "papermill": {
     "duration": 25.803068,
     "end_time": "2024-03-28T18:56:52.610477",
     "exception": false,
     "start_time": "2024-03-28T18:56:26.807409",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.0.5'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras\n",
    "from keras import ops\n",
    "from keras_cv.models import VideoSwinBackbone\n",
    "from keras_cv.models import VideoClassifier\n",
    "\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "af7810e0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:56:52.629367Z",
     "iopub.status.busy": "2024-03-28T18:56:52.628675Z",
     "iopub.status.idle": "2024-03-28T18:56:52.635618Z",
     "shell.execute_reply": "2024-03-28T18:56:52.634364Z"
    },
    "papermill": {
     "duration": 0.019551,
     "end_time": "2024-03-28T18:56:52.638019",
     "exception": false,
     "start_time": "2024-03-28T18:56:52.618468",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def vswin_tiny():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=96,\n",
    "        depths=[2, 2, 6, 2],\n",
    "        num_heads=[3, 6, 12, 24],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    keras_model = VideoClassifier(\n",
    "        backbone=backbone,\n",
    "        num_classes=400,\n",
    "        activation=None,\n",
    "        pooling='avg',\n",
    "    )\n",
    "    keras_model.load_weights(\n",
    "        '/kaggle/input/videoswin/keras/tiny/1/videoswin_tiny_kinetics400_classifier.weights.h5'\n",
    "    )\n",
    "    return keras_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f27545dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:56:52.655360Z",
     "iopub.status.busy": "2024-03-28T18:56:52.654951Z",
     "iopub.status.idle": "2024-03-28T18:56:52.661588Z",
     "shell.execute_reply": "2024-03-28T18:56:52.660250Z"
    },
    "papermill": {
     "duration": 0.018416,
     "end_time": "2024-03-28T18:56:52.664262",
     "exception": false,
     "start_time": "2024-03-28T18:56:52.645846",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def vswin_small():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=96,\n",
    "        depths=[2, 2, 18, 2],\n",
    "        num_heads=[3, 6, 12, 24],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    keras_model = VideoClassifier(\n",
    "        backbone=backbone,\n",
    "        num_classes=400,\n",
    "        activation=None,\n",
    "        pooling='avg',\n",
    "    )\n",
    "    keras_model.load_weights(\n",
    "        '/kaggle/input/videoswin/keras/small/1/videoswin_small_kinetics400_classifier.weights.h5'\n",
    "    )\n",
    "    return keras_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0a6bcb76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:56:52.682793Z",
     "iopub.status.busy": "2024-03-28T18:56:52.681811Z",
     "iopub.status.idle": "2024-03-28T18:56:52.688494Z",
     "shell.execute_reply": "2024-03-28T18:56:52.687403Z"
    },
    "papermill": {
     "duration": 0.0189,
     "end_time": "2024-03-28T18:56:52.690784",
     "exception": false,
     "start_time": "2024-03-28T18:56:52.671884",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def vswin_base():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=128,\n",
    "        depths=[2, 2, 18, 2],\n",
    "        num_heads=[4, 8, 16, 32],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    keras_model = VideoClassifier(\n",
    "        backbone=backbone,\n",
    "        num_classes=400,\n",
    "        activation=None,\n",
    "        pooling='avg',\n",
    "    )\n",
    "    keras_model.load_weights(\n",
    "        '/kaggle/input/videoswin/keras/base/1/videoswin_base_kinetics400_classifier.weights.h5'\n",
    "    )\n",
    "    return keras_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25687c67",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:56:52.709040Z",
     "iopub.status.busy": "2024-03-28T18:56:52.708618Z",
     "iopub.status.idle": "2024-03-28T18:57:07.215996Z",
     "shell.execute_reply": "2024-03-28T18:57:07.215121Z"
    },
    "papermill": {
     "duration": 14.519874,
     "end_time": "2024-03-28T18:57:07.218056",
     "exception": false,
     "start_time": "2024-03-28T18:56:52.698182",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"video_classifier\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"video_classifier\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ videos (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">224</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">224</span>,   │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│                                 │ <span style=\"color: #00af00; text-decoration-color: #00af00\">3</span>)                     │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ video_swin_backbone             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)  │    <span style=\"color: #00af00; text-decoration-color: #00af00\">27,850,470</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinBackbone</span>)             │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ avg_pool                        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalAveragePooling3D</span>)        │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ predictions (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">400</span>)            │       <span style=\"color: #00af00; text-decoration-color: #00af00\">307,600</span> │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ videos (\u001b[38;5;33mInputLayer\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m, \u001b[38;5;34m224\u001b[0m, \u001b[38;5;34m224\u001b[0m,   │             \u001b[38;5;34m0\u001b[0m │\n",
       "│                                 │ \u001b[38;5;34m3\u001b[0m)                     │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ video_swin_backbone             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m768\u001b[0m)  │    \u001b[38;5;34m27,850,470\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinBackbone\u001b[0m)             │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ avg_pool                        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n",
       "│ (\u001b[38;5;33mGlobalAveragePooling3D\u001b[0m)        │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ predictions (\u001b[38;5;33mDense\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m400\u001b[0m)            │       \u001b[38;5;34m307,600\u001b[0m │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">28,158,070</span> (107.41 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m28,158,070\u001b[0m (107.41 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">28,158,070</span> (107.41 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m28,158,070\u001b[0m (107.41 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "keras_models = [vswin_tiny(), vswin_small(), vswin_base()]\n",
    "keras_models[0].summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "544542ba",
   "metadata": {
    "papermill": {
     "duration": 0.00794,
     "end_time": "2024-03-28T18:57:07.234196",
     "exception": false,
     "start_time": "2024-03-28T18:57:07.226256",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TorchVision: Video Swin : Pretrained: ImageNet 1K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a8fe6d48",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:57:07.254724Z",
     "iopub.status.busy": "2024-03-28T18:57:07.253993Z",
     "iopub.status.idle": "2024-03-28T18:57:07.628853Z",
     "shell.execute_reply": "2024-03-28T18:57:07.627884Z"
    },
    "papermill": {
     "duration": 0.388364,
     "end_time": "2024-03-28T18:57:07.631527",
     "exception": false,
     "start_time": "2024-03-28T18:57:07.243163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchinfo import summary\n",
    "from torchvision.models.video import Swin3D_T_Weights, Swin3D_S_Weights, Swin3D_B_Weights\n",
    "\n",
    "def torch_vswin_tiny():\n",
    "    torch_model = torchvision.models.video.swin3d_t(\n",
    "        weights=Swin3D_T_Weights.KINETICS400_V1\n",
    "    ).eval()\n",
    "    return torch_model\n",
    "\n",
    "def torch_vswin_small():\n",
    "    torch_model = torchvision.models.video.swin3d_s(\n",
    "        weights=Swin3D_S_Weights.KINETICS400_V1\n",
    "    ).eval()\n",
    "    return torch_model\n",
    "\n",
    "def torch_vswin_base():\n",
    "    torch_model = torchvision.models.video.swin3d_b(\n",
    "        weights=Swin3D_B_Weights.KINETICS400_V1\n",
    "    ).eval()\n",
    "    return torch_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "baa9604b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:57:07.649608Z",
     "iopub.status.busy": "2024-03-28T18:57:07.649212Z",
     "iopub.status.idle": "2024-03-28T18:57:34.795055Z",
     "shell.execute_reply": "2024-03-28T18:57:34.794149Z"
    },
    "papermill": {
     "duration": 27.157606,
     "end_time": "2024-03-28T18:57:34.797479",
     "exception": false,
     "start_time": "2024-03-28T18:57:07.639873",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/swin3d_t-7615ae03.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_t-7615ae03.pth\n",
      "100%|██████████| 122M/122M [00:02<00:00, 54.0MB/s]\n",
      "Downloading: \"https://download.pytorch.org/models/swin3d_s-da41c237.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_s-da41c237.pth\n",
      "100%|██████████| 218M/218M [00:04<00:00, 55.4MB/s]\n",
      "Downloading: \"https://download.pytorch.org/models/swin3d_b_1k-24f7c7c6.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_b_1k-24f7c7c6.pth\n",
      "100%|██████████| 364M/364M [00:06<00:00, 57.0MB/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "=========================================================================================================\n",
       "Layer (type:depth-idx)                                  Output Shape              Param #\n",
       "=========================================================================================================\n",
       "SwinTransformer3d                                       [1, 400]                  --\n",
       "├─PatchEmbed3d: 1-1                                     [1, 16, 56, 56, 96]       --\n",
       "│    └─Conv3d: 2-1                                      [1, 96, 16, 56, 56]       9,312\n",
       "│    └─LayerNorm: 2-2                                   [1, 16, 56, 56, 96]       192\n",
       "├─Dropout: 1-2                                          [1, 16, 56, 56, 96]       --\n",
       "├─Sequential: 1-3                                       [1, 16, 7, 7, 768]        --\n",
       "│    └─Sequential: 2-3                                  [1, 16, 56, 56, 96]       --\n",
       "│    │    └─SwinTransformerBlock: 3-1                   [1, 16, 56, 56, 96]       119,445\n",
       "│    │    └─SwinTransformerBlock: 3-2                   [1, 16, 56, 56, 96]       119,445\n",
       "│    └─PatchMerging: 2-4                                [1, 16, 28, 28, 192]      --\n",
       "│    │    └─LayerNorm: 3-3                              [1, 16, 28, 28, 384]      768\n",
       "│    │    └─Linear: 3-4                                 [1, 16, 28, 28, 192]      73,728\n",
       "│    └─Sequential: 2-5                                  [1, 16, 28, 28, 192]      --\n",
       "│    │    └─SwinTransformerBlock: 3-5                   [1, 16, 28, 28, 192]      460,074\n",
       "│    │    └─SwinTransformerBlock: 3-6                   [1, 16, 28, 28, 192]      460,074\n",
       "│    └─PatchMerging: 2-6                                [1, 16, 14, 14, 384]      --\n",
       "│    │    └─LayerNorm: 3-7                              [1, 16, 14, 14, 768]      1,536\n",
       "│    │    └─Linear: 3-8                                 [1, 16, 14, 14, 384]      294,912\n",
       "│    └─Sequential: 2-7                                  [1, 16, 14, 14, 384]      --\n",
       "│    │    └─SwinTransformerBlock: 3-9                   [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-10                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-11                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-12                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-13                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    │    └─SwinTransformerBlock: 3-14                  [1, 16, 14, 14, 384]      1,804,884\n",
       "│    └─PatchMerging: 2-8                                [1, 16, 7, 7, 768]        --\n",
       "│    │    └─LayerNorm: 3-15                             [1, 16, 7, 7, 1536]       3,072\n",
       "│    │    └─Linear: 3-16                                [1, 16, 7, 7, 768]        1,179,648\n",
       "│    └─Sequential: 2-9                                  [1, 16, 7, 7, 768]        --\n",
       "│    │    └─SwinTransformerBlock: 3-17                  [1, 16, 7, 7, 768]        7,148,712\n",
       "│    │    └─SwinTransformerBlock: 3-18                  [1, 16, 7, 7, 768]        7,148,712\n",
       "├─LayerNorm: 1-4                                        [1, 16, 7, 7, 768]        1,536\n",
       "├─AdaptiveAvgPool3d: 1-5                                [1, 768, 1, 1, 1]         --\n",
       "├─Linear: 1-6                                           [1, 400]                  307,600\n",
       "=========================================================================================================\n",
       "Total params: 28,158,070\n",
       "Trainable params: 28,158,070\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (M): 486.39\n",
       "=========================================================================================================\n",
       "Input size (MB): 19.27\n",
       "Forward/backward pass size (MB): 1464.34\n",
       "Params size (MB): 76.66\n",
       "Estimated Total Size (MB): 1560.26\n",
       "========================================================================================================="
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_models = [torch_vswin_tiny(), torch_vswin_small(), torch_vswin_base()]\n",
    "summary(\n",
    "    torch_models[0], input_size=(1, 3, 32, 224, 224)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "967ea820",
   "metadata": {
    "papermill": {
     "duration": 0.015667,
     "end_time": "2024-03-28T18:57:34.828907",
     "exception": false,
     "start_time": "2024-03-28T18:57:34.813240",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "45dcf674",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:57:34.861533Z",
     "iopub.status.busy": "2024-03-28T18:57:34.860787Z",
     "iopub.status.idle": "2024-03-28T18:57:35.023585Z",
     "shell.execute_reply": "2024-03-28T18:57:35.022058Z"
    },
    "papermill": {
     "duration": 0.182132,
     "end_time": "2024-03-28T18:57:35.025987",
     "exception": false,
     "start_time": "2024-03-28T18:57:34.843855",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 32, 224, 224, 3]) torch.Size([1, 3, 32, 224, 224])\n"
     ]
    }
   ],
   "source": [
    "common_input = np.random.normal(0, 1, (1, 32, 224, 224, 3)).astype('float32')\n",
    "keras_input = ops.array(common_input)\n",
    "torch_input = torch.from_numpy(common_input.transpose(0, 4, 1, 2, 3))\n",
    "print(keras_input.shape, torch_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1758718d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:57:35.058379Z",
     "iopub.status.busy": "2024-03-28T18:57:35.057419Z",
     "iopub.status.idle": "2024-03-28T18:57:35.065129Z",
     "shell.execute_reply": "2024-03-28T18:57:35.063980Z"
    },
    "papermill": {
     "duration": 0.026627,
     "end_time": "2024-03-28T18:57:35.067490",
     "exception": false,
     "start_time": "2024-03-28T18:57:35.040863",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def logit_checking(keras_model, torch_model):\n",
    "    # forward pass\n",
    "    keras_predict = keras_model(keras_input)\n",
    "    torch_predict = torch_model(torch_input)\n",
    "    print(keras_predict.shape, torch_predict.shape)\n",
    "    print('keras logits: ', keras_predict[0, :5])\n",
    "    print('torch logits: ', torch_predict[0, :5], end='\\n')\n",
    "    np.testing.assert_allclose(\n",
    "        keras_predict.detach().numpy(),\n",
    "        torch_predict.detach().numpy(),\n",
    "        1e-5, 1e-5\n",
    "    )\n",
    "    del keras_model \n",
    "    del torch_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "68a1e59a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:57:35.100767Z",
     "iopub.status.busy": "2024-03-28T18:57:35.100079Z",
     "iopub.status.idle": "2024-03-28T18:59:16.615601Z",
     "shell.execute_reply": "2024-03-28T18:59:16.613891Z"
    },
    "papermill": {
     "duration": 101.535287,
     "end_time": "2024-03-28T18:59:16.618485",
     "exception": false,
     "start_time": "2024-03-28T18:57:35.083198",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 400]) torch.Size([1, 400])\n",
      "keras logits:  tensor([-0.0906,  1.2267,  1.1639, -0.3530, -1.5449], grad_fn=<SliceBackward0>)\n",
      "torch logits:  tensor([-0.0906,  1.2267,  1.1639, -0.3530, -1.5449], grad_fn=<SliceBackward0>)\n",
      "torch.Size([1, 400]) torch.Size([1, 400])\n",
      "keras logits:  tensor([ 0.6399,  1.2136,  0.9395, -0.4962, -1.9626], grad_fn=<SliceBackward0>)\n",
      "torch logits:  tensor([ 0.6399,  1.2136,  0.9395, -0.4962, -1.9626], grad_fn=<SliceBackward0>)\n",
      "torch.Size([1, 400]) torch.Size([1, 400])\n",
      "keras logits:  tensor([ 1.1572,  0.0092,  0.0929, -1.8786, -2.8799], grad_fn=<SliceBackward0>)\n",
      "torch logits:  tensor([ 1.1572,  0.0092,  0.0929, -1.8786, -2.8799], grad_fn=<SliceBackward0>)\n"
     ]
    }
   ],
   "source": [
    "for km, tm in zip(keras_models, torch_models):\n",
    "    logit_checking(\n",
    "        km, tm\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8b7aefae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:59:16.653715Z",
     "iopub.status.busy": "2024-03-28T18:59:16.653289Z",
     "iopub.status.idle": "2024-03-28T18:59:17.266830Z",
     "shell.execute_reply": "2024-03-28T18:59:17.265516Z"
    },
    "papermill": {
     "duration": 0.633776,
     "end_time": "2024-03-28T18:59:17.269193",
     "exception": false,
     "start_time": "2024-03-28T18:59:16.635417",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e73db7f4",
   "metadata": {
    "papermill": {
     "duration": 0.01687,
     "end_time": "2024-03-28T18:59:17.302135",
     "exception": false,
     "start_time": "2024-03-28T18:59:17.285265",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Keras: Video Swin Base - Pretrained: ImageNet 22K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ef9221ed",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:59:17.334873Z",
     "iopub.status.busy": "2024-03-28T18:59:17.334394Z",
     "iopub.status.idle": "2024-03-28T18:59:17.341984Z",
     "shell.execute_reply": "2024-03-28T18:59:17.340602Z"
    },
    "papermill": {
     "duration": 0.027379,
     "end_time": "2024-03-28T18:59:17.344960",
     "exception": false,
     "start_time": "2024-03-28T18:59:17.317581",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def vswin_base():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=128,\n",
    "        depths=[2, 2, 18, 2],\n",
    "        num_heads=[4, 8, 16, 32],\n",
    "        include_rescaling=False, \n",
    "    )\n",
    "    keras_model = VideoClassifier(\n",
    "        backbone=backbone,\n",
    "        num_classes=400,\n",
    "        activation=None,\n",
    "        pooling='avg',\n",
    "    )\n",
    "    keras_model.load_weights(\n",
    "        '/kaggle/input/videoswin/keras/base/1/videoswin_base_kinetics400_imagenet22k_classifier.weights.h5'\n",
    "    )\n",
    "    return keras_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1f8d0375",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:59:17.380760Z",
     "iopub.status.busy": "2024-03-28T18:59:17.380288Z",
     "iopub.status.idle": "2024-03-28T18:59:23.407797Z",
     "shell.execute_reply": "2024-03-28T18:59:23.406765Z"
    },
    "papermill": {
     "duration": 6.048239,
     "end_time": "2024-03-28T18:59:23.411009",
     "exception": false,
     "start_time": "2024-03-28T18:59:17.362770",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_models = vswin_base()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "25e03a40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:59:23.444463Z",
     "iopub.status.busy": "2024-03-28T18:59:23.444048Z",
     "iopub.status.idle": "2024-03-28T18:59:33.888855Z",
     "shell.execute_reply": "2024-03-28T18:59:33.887528Z"
    },
    "papermill": {
     "duration": 10.465804,
     "end_time": "2024-03-28T18:59:33.892692",
     "exception": false,
     "start_time": "2024-03-28T18:59:23.426888",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/swin3d_b_22k-7c6ae6fa.pth\" to /root/.cache/torch/hub/checkpoints/swin3d_b_22k-7c6ae6fa.pth\n",
      "100%|██████████| 364M/364M [00:07<00:00, 51.8MB/s]\n"
     ]
    }
   ],
   "source": [
    "import torchvision\n",
    "from torchvision.models.video import Swin3D_B_Weights\n",
    "\n",
    "torch_model = torchvision.models.video.swin3d_b(\n",
    "    weights=Swin3D_B_Weights.KINETICS400_IMAGENET22K_V1\n",
    ").eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "908ae048",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-28T18:59:33.936777Z",
     "iopub.status.busy": "2024-03-28T18:59:33.935944Z",
     "iopub.status.idle": "2024-03-28T19:00:04.235561Z",
     "shell.execute_reply": "2024-03-28T19:00:04.234520Z"
    },
    "papermill": {
     "duration": 30.323177,
     "end_time": "2024-03-28T19:00:04.238763",
     "exception": false,
     "start_time": "2024-03-28T18:59:33.915586",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 400]) torch.Size([1, 400])\n",
      "keras logits:  tensor([ 0.2773,  0.8488,  1.4034, -1.0703, -1.4610], grad_fn=<SliceBackward0>)\n",
      "torch logits:  tensor([ 0.2773,  0.8488,  1.4034, -1.0703, -1.4610], grad_fn=<SliceBackward0>)\n"
     ]
    }
   ],
   "source": [
    "logit_checking(\n",
    "    keras_models, torch_model\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "isSourceIdPinned": true,
     "modelInstanceId": 17431,
     "sourceId": 21048,
     "sourceType": "modelInstanceVersion"
    },
    {
     "isSourceIdPinned": true,
     "modelInstanceId": 17474,
     "sourceId": 21097,
     "sourceType": "modelInstanceVersion"
    },
    {
     "isSourceIdPinned": true,
     "modelInstanceId": 17533,
     "sourceId": 21184,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30673,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 257.248119,
   "end_time": "2024-03-28T19:00:07.884277",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-03-28T18:55:50.636158",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}