student_model_builder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>MODEL BUILDER</h1>\n",
    "<p>The purpose of this notebook is to build the proposed model, the dual fused EfficientNet Entry-Block (ENEB) with a Modified Residual Skip Block (MReSBlock)</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOAD DEPENDENCIES\n",
    "import os\n",
    "import logging\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras import applications, Model, layers\n",
    "\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "from tensorflow.keras.layers import Layer, Conv2D, GlobalAveragePooling2D, Dense, Dropout\n",
    "from tensorflow.keras.layers import Input, DepthwiseConv2D, Activation, Add, BatchNormalization\n",
    "\n",
    "from tensorflow.keras.applications.efficientnet import EfficientNetB0 as selected_model, preprocess_input\n",
    "\n",
    "model_architecture = \"proposed_model\"\n",
    "\n",
    "activation_setting = tf.keras.activations.selu\n",
    "\n",
    "#PREVENT ERROR UNCESSARY MESSAGES\n",
    "tf.get_logger().setLevel(logging.ERROR)\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Prepare essentials</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Custom Functions\n",
    "data_path = \"dataset\"\n",
    "model_kind = \"proposed_model\"\n",
    "model_path = \"models/\" + model_kind + '/'\n",
    "\n",
    "#Save Model Function\n",
    "def save_m(directory, model):\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    model.save(directory + '/' + model_architecture + '.h5')\n",
    "    print(\"model saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Input size is set to  KerasTensor(type_spec=TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='input_1'), name='input_1', description=\"created by layer 'input_1'\")\n"
     ]
    }
   ],
   "source": [
    "#LOAD THE DATA\n",
    "\n",
    "img_rows, img_cols = 224, 224\n",
    "input_shape = (img_rows,img_cols,3)\n",
    "model_input = Input(shape=input_shape)\n",
    "\n",
    "print(\"The Input size is set to \", model_input) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>ENEB Builder</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PLEASE CHECK THE ENTIRE MODEL UP TO THE END\n",
      "Model: \"proposed_model-a\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1-a (InputLayer)          [(None, 224, 224, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "rescaling-a (Rescaling)         (None, 224, 224, 3)  0           input_1-a[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "normalization-a (Normalization) (None, 224, 224, 3)  7           rescaling-a[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv_pad-a (ZeroPadding2D) (None, 225, 225, 3)  0           normalization-a[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv-a (Conv2D)            (None, 112, 112, 32) 864         stem_conv_pad-a[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "stem_bn-a (BatchNormalization)  (None, 112, 112, 32) 128         stem_conv-a[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "stem_activation-a (Activation)  (None, 112, 112, 32) 0           stem_bn-a[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "block1a_dwconv-a (DepthwiseConv (None, 112, 112, 32) 288         stem_activation-a[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block1a_bn-a (BatchNormalizatio (None, 112, 112, 32) 128         block1a_dwconv-a[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1a_activation-a (Activatio (None, 112, 112, 32) 0           block1a_bn-a[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_squeeze-a (GlobalAve (None, 32)           0           block1a_activation-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reshape-a (Reshape)  (None, 1, 1, 32)     0           block1a_se_squeeze-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reduce-a (Conv2D)    (None, 1, 1, 8)      264         block1a_se_reshape-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_expand-a (Conv2D)    (None, 1, 1, 32)     288         block1a_se_reduce-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_excite-a (Multiply)  (None, 112, 112, 32) 0           block1a_activation-a[0][0]       \n",
      "                                                                 block1a_se_expand-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_conv-a (Conv2D) (None, 112, 112, 16) 512         block1a_se_excite-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_bn-a (BatchNorm (None, 112, 112, 16) 64          block1a_project_conv-a[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_conv-a (Conv2D)  (None, 112, 112, 96) 1536        block1a_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_bn-a (BatchNorma (None, 112, 112, 96) 384         block2a_expand_conv-a[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_activation-a (Ac (None, 112, 112, 96) 0           block2a_expand_bn-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv_pad-a (ZeroPaddi (None, 113, 113, 96) 0           block2a_expand_activation-a[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv-a (DepthwiseConv (None, 56, 56, 96)   864         block2a_dwconv_pad-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_bn-a (BatchNormalizatio (None, 56, 56, 96)   384         block2a_dwconv-a[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2a_activation-a (Activatio (None, 56, 56, 96)   0           block2a_bn-a[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_squeeze-a (GlobalAve (None, 96)           0           block2a_activation-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reshape-a (Reshape)  (None, 1, 1, 96)     0           block2a_se_squeeze-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reduce-a (Conv2D)    (None, 1, 1, 4)      388         block2a_se_reshape-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_expand-a (Conv2D)    (None, 1, 1, 96)     480         block2a_se_reduce-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_excite-a (Multiply)  (None, 56, 56, 96)   0           block2a_activation-a[0][0]       \n",
      "                                                                 block2a_se_expand-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_conv-a (Conv2D) (None, 56, 56, 24)   2304        block2a_se_excite-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_bn-a (BatchNorm (None, 56, 56, 24)   96          block2a_project_conv-a[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_conv-a (Conv2D)  (None, 56, 56, 144)  3456        block2a_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_bn-a (BatchNorma (None, 56, 56, 144)  576         block2b_expand_conv-a[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_activation-a (Ac (None, 56, 56, 144)  0           block2b_expand_bn-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_dwconv-a (DepthwiseConv (None, 56, 56, 144)  1296        block2b_expand_activation-a[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2b_bn-a (BatchNormalizatio (None, 56, 56, 144)  576         block2b_dwconv-a[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2b_activation-a (Activatio (None, 56, 56, 144)  0           block2b_bn-a[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_squeeze-a (GlobalAve (None, 144)          0           block2b_activation-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reshape-a (Reshape)  (None, 1, 1, 144)    0           block2b_se_squeeze-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reduce-a (Conv2D)    (None, 1, 1, 6)      870         block2b_se_reshape-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_expand-a (Conv2D)    (None, 1, 1, 144)    1008        block2b_se_reduce-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_excite-a (Multiply)  (None, 56, 56, 144)  0           block2b_activation-a[0][0]       \n",
      "                                                                 block2b_se_expand-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_conv-a (Conv2D) (None, 56, 56, 24)   3456        block2b_se_excite-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_bn-a (BatchNorm (None, 56, 56, 24)   96          block2b_project_conv-a[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_drop-a (Dropout)        (None, 56, 56, 24)   0           block2b_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_add-a (Add)             (None, 56, 56, 24)   0           block2b_drop-a[0][0]             \n",
      "                                                                 block2a_project_bn-a[0][0]       \n",
      "==================================================================================================\n",
      "Total params: 20,313\n",
      "Trainable params: 19,090\n",
      "Non-trainable params: 1,223\n",
      "__________________________________________________________________________________________________\n",
      "Model successfully built!\n"
     ]
    }
   ],
   "source": [
    "# Builder \n",
    "\n",
    "#TRANSFER LEARNING\n",
    "def model_builder_a(model_input):\n",
    "    model_builder_a = selected_model(weights='imagenet', include_top=False, input_tensor=model_input)\n",
    "    \n",
    "#FINE TUNING\n",
    "    for layer in model_builder_a.layers:\n",
    "        layer.trainable = True\n",
    "    for BatchNormalization in model_builder_a.layers:\n",
    "        BatchNormalization.trainable = True\n",
    "        \n",
    "    for layer in model_builder_a.layers:\n",
    "        layer._name = layer.name + '-a'\n",
    "        \n",
    "    x = model_builder_a.layers[-192].output\n",
    "\n",
    "    model_a = Model(inputs=model_builder_a.input, outputs=x, name=model_architecture + '-a')\n",
    "    return model_a\n",
    "\n",
    "#INITIALIZE THE MODEL\n",
    "model_a = model_builder_a(model_input)\n",
    "\n",
    "#PLOT THE MODEL STRUCTURE\n",
    "print(\"PLEASE CHECK THE ENTIRE MODEL UP TO THE END\")\n",
    "model_a.summary()\n",
    "print(\"Model successfully built!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PLEASE CHECK THE ENTIRE MODEL UP TO THE END\n",
      "Model: \"proposed_model-b\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1-a-b (InputLayer)        [(None, 224, 224, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "rescaling_1-b (Rescaling)       (None, 224, 224, 3)  0           input_1-a-b[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "normalization_1-b (Normalizatio (None, 224, 224, 3)  7           rescaling_1-b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv_pad-b (ZeroPadding2D) (None, 225, 225, 3)  0           normalization_1-b[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv-b (Conv2D)            (None, 112, 112, 32) 864         stem_conv_pad-b[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "stem_bn-b (BatchNormalization)  (None, 112, 112, 32) 128         stem_conv-b[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "stem_activation-b (Activation)  (None, 112, 112, 32) 0           stem_bn-b[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "block1a_dwconv-b (DepthwiseConv (None, 112, 112, 32) 288         stem_activation-b[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block1a_bn-b (BatchNormalizatio (None, 112, 112, 32) 128         block1a_dwconv-b[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1a_activation-b (Activatio (None, 112, 112, 32) 0           block1a_bn-b[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_squeeze-b (GlobalAve (None, 32)           0           block1a_activation-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reshape-b (Reshape)  (None, 1, 1, 32)     0           block1a_se_squeeze-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reduce-b (Conv2D)    (None, 1, 1, 8)      264         block1a_se_reshape-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_expand-b (Conv2D)    (None, 1, 1, 32)     288         block1a_se_reduce-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_excite-b (Multiply)  (None, 112, 112, 32) 0           block1a_activation-b[0][0]       \n",
      "                                                                 block1a_se_expand-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_conv-b (Conv2D) (None, 112, 112, 16) 512         block1a_se_excite-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_bn-b (BatchNorm (None, 112, 112, 16) 64          block1a_project_conv-b[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_conv-b (Conv2D)  (None, 112, 112, 96) 1536        block1a_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_bn-b (BatchNorma (None, 112, 112, 96) 384         block2a_expand_conv-b[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_activation-b (Ac (None, 112, 112, 96) 0           block2a_expand_bn-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv_pad-b (ZeroPaddi (None, 113, 113, 96) 0           block2a_expand_activation-b[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv-b (DepthwiseConv (None, 56, 56, 96)   864         block2a_dwconv_pad-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_bn-b (BatchNormalizatio (None, 56, 56, 96)   384         block2a_dwconv-b[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2a_activation-b (Activatio (None, 56, 56, 96)   0           block2a_bn-b[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_squeeze-b (GlobalAve (None, 96)           0           block2a_activation-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reshape-b (Reshape)  (None, 1, 1, 96)     0           block2a_se_squeeze-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reduce-b (Conv2D)    (None, 1, 1, 4)      388         block2a_se_reshape-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_expand-b (Conv2D)    (None, 1, 1, 96)     480         block2a_se_reduce-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_excite-b (Multiply)  (None, 56, 56, 96)   0           block2a_activation-b[0][0]       \n",
      "                                                                 block2a_se_expand-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_conv-b (Conv2D) (None, 56, 56, 24)   2304        block2a_se_excite-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_bn-b (BatchNorm (None, 56, 56, 24)   96          block2a_project_conv-b[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_conv-b (Conv2D)  (None, 56, 56, 144)  3456        block2a_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_bn-b (BatchNorma (None, 56, 56, 144)  576         block2b_expand_conv-b[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_activation-b (Ac (None, 56, 56, 144)  0           block2b_expand_bn-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_dwconv-b (DepthwiseConv (None, 56, 56, 144)  1296        block2b_expand_activation-b[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2b_bn-b (BatchNormalizatio (None, 56, 56, 144)  576         block2b_dwconv-b[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2b_activation-b (Activatio (None, 56, 56, 144)  0           block2b_bn-b[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_squeeze-b (GlobalAve (None, 144)          0           block2b_activation-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reshape-b (Reshape)  (None, 1, 1, 144)    0           block2b_se_squeeze-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reduce-b (Conv2D)    (None, 1, 1, 6)      870         block2b_se_reshape-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_expand-b (Conv2D)    (None, 1, 1, 144)    1008        block2b_se_reduce-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_excite-b (Multiply)  (None, 56, 56, 144)  0           block2b_activation-b[0][0]       \n",
      "                                                                 block2b_se_expand-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_conv-b (Conv2D) (None, 56, 56, 24)   3456        block2b_se_excite-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_bn-b (BatchNorm (None, 56, 56, 24)   96          block2b_project_conv-b[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_drop-b (Dropout)        (None, 56, 56, 24)   0           block2b_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_add-b (Add)             (None, 56, 56, 24)   0           block2b_drop-b[0][0]             \n",
      "                                                                 block2a_project_bn-b[0][0]       \n",
      "==================================================================================================\n",
      "Total params: 20,313\n",
      "Trainable params: 19,090\n",
      "Non-trainable params: 1,223\n",
      "__________________________________________________________________________________________________\n",
      "Model successfully built!\n"
     ]
    }
   ],
   "source": [
    "# Builder\n",
    "\n",
    "#TRANSFER LEARNING\n",
    "def model_builder_b(model_input):\n",
    "    model_builder_b = selected_model(weights='imagenet', include_top=False, input_tensor=model_input)\n",
    "    \n",
    "#FINE TUNING\n",
    "    for layer in model_builder_b.layers:\n",
    "        layer.trainable = True\n",
    "    for BatchNormalization in model_builder_b.layers:\n",
    "        BatchNormalization.trainable = True\n",
    "    for layer in model_builder_b.layers:\n",
    "        layer._name = layer.name + '-b'\n",
    "        \n",
    "    x = model_builder_b.layers[-192].output\n",
    "\n",
    "    model_b = Model(inputs=model_builder_b.input, outputs=x, name=model_architecture + '-b')\n",
    "    return model_b\n",
    "\n",
    "#INITIALIZE THE MODEL\n",
    "model_b = model_builder_b(model_input)\n",
    "\n",
    "#PLOT THE MODEL STRUCTURE\n",
    "print(\"PLEASE CHECK THE ENTIRE MODEL UP TO THE END\")\n",
    "model_b.summary()\n",
    "print(\"Model successfully built!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Fusion of the ENEB architectures</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accomplished Pre-training and ready for fusion\n"
     ]
    }
   ],
   "source": [
    "#RE-INITIALIZE FOR FUSION\n",
    "model_a = model_builder_a(model_input)\n",
    "model_b = model_builder_b(model_input)\n",
    "\n",
    "print(\"Accomplished Pre-training and ready for fusion\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fusion success!\n",
      "Ready to connect with its ending layers!\n"
     ]
    }
   ],
   "source": [
    "#FUSE THE MODELS INTO A SINGLE PIPELINE\n",
    "\n",
    "models = [model_a, \n",
    "          model_b]\n",
    "\n",
    "print(\"Fusion success!\")\n",
    "print(\"Ready to connect with its ending layers!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Build the final model attached with the MReSBlock</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "PLEASE CHECK THE MODEL UP TO THE END\n",
      "\n",
      "\n",
      "\n",
      "Complete and ready for compilation and training!\n",
      "The error is caused because of the modifications made from the original layers\n",
      "model saved\n",
      "Model: \"proposed_model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1-a-b-a-b (InputLayer)    [(None, 224, 224, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "rescaling_2-a (Rescaling)       (None, 224, 224, 3)  0           input_1-a-b-a-b[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "rescaling_3-b (Rescaling)       (None, 224, 224, 3)  0           input_1-a-b-a-b[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "normalization_2-a (Normalizatio (None, 224, 224, 3)  7           rescaling_2-a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "normalization_3-b (Normalizatio (None, 224, 224, 3)  7           rescaling_3-b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv_pad-a (ZeroPadding2D) (None, 225, 225, 3)  0           normalization_2-a[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv_pad-b (ZeroPadding2D) (None, 225, 225, 3)  0           normalization_3-b[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv-a (Conv2D)            (None, 112, 112, 32) 864         stem_conv_pad-a[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "stem_conv-b (Conv2D)            (None, 112, 112, 32) 864         stem_conv_pad-b[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "stem_bn-a (BatchNormalization)  (None, 112, 112, 32) 128         stem_conv-a[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "stem_bn-b (BatchNormalization)  (None, 112, 112, 32) 128         stem_conv-b[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "stem_activation-a (Activation)  (None, 112, 112, 32) 0           stem_bn-a[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "stem_activation-b (Activation)  (None, 112, 112, 32) 0           stem_bn-b[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "block1a_dwconv-a (DepthwiseConv (None, 112, 112, 32) 288         stem_activation-a[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block1a_dwconv-b (DepthwiseConv (None, 112, 112, 32) 288         stem_activation-b[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block1a_bn-a (BatchNormalizatio (None, 112, 112, 32) 128         block1a_dwconv-a[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1a_bn-b (BatchNormalizatio (None, 112, 112, 32) 128         block1a_dwconv-b[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1a_activation-a (Activatio (None, 112, 112, 32) 0           block1a_bn-a[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1a_activation-b (Activatio (None, 112, 112, 32) 0           block1a_bn-b[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_squeeze-a (GlobalAve (None, 32)           0           block1a_activation-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_squeeze-b (GlobalAve (None, 32)           0           block1a_activation-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reshape-a (Reshape)  (None, 1, 1, 32)     0           block1a_se_squeeze-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reshape-b (Reshape)  (None, 1, 1, 32)     0           block1a_se_squeeze-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reduce-a (Conv2D)    (None, 1, 1, 8)      264         block1a_se_reshape-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_reduce-b (Conv2D)    (None, 1, 1, 8)      264         block1a_se_reshape-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_expand-a (Conv2D)    (None, 1, 1, 32)     288         block1a_se_reduce-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_expand-b (Conv2D)    (None, 1, 1, 32)     288         block1a_se_reduce-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_excite-a (Multiply)  (None, 112, 112, 32) 0           block1a_activation-a[0][0]       \n",
      "                                                                 block1a_se_expand-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_se_excite-b (Multiply)  (None, 112, 112, 32) 0           block1a_activation-b[0][0]       \n",
      "                                                                 block1a_se_expand-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_conv-a (Conv2D) (None, 112, 112, 16) 512         block1a_se_excite-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_conv-b (Conv2D) (None, 112, 112, 16) 512         block1a_se_excite-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_bn-a (BatchNorm (None, 112, 112, 16) 64          block1a_project_conv-a[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block1a_project_bn-b (BatchNorm (None, 112, 112, 16) 64          block1a_project_conv-b[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_conv-a (Conv2D)  (None, 112, 112, 96) 1536        block1a_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_conv-b (Conv2D)  (None, 112, 112, 96) 1536        block1a_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_bn-a (BatchNorma (None, 112, 112, 96) 384         block2a_expand_conv-a[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_bn-b (BatchNorma (None, 112, 112, 96) 384         block2a_expand_conv-b[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_activation-a (Ac (None, 112, 112, 96) 0           block2a_expand_bn-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_expand_activation-b (Ac (None, 112, 112, 96) 0           block2a_expand_bn-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv_pad-a (ZeroPaddi (None, 113, 113, 96) 0           block2a_expand_activation-a[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv_pad-b (ZeroPaddi (None, 113, 113, 96) 0           block2a_expand_activation-b[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv-a (DepthwiseConv (None, 56, 56, 96)   864         block2a_dwconv_pad-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_dwconv-b (DepthwiseConv (None, 56, 56, 96)   864         block2a_dwconv_pad-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_bn-a (BatchNormalizatio (None, 56, 56, 96)   384         block2a_dwconv-a[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2a_bn-b (BatchNormalizatio (None, 56, 56, 96)   384         block2a_dwconv-b[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2a_activation-a (Activatio (None, 56, 56, 96)   0           block2a_bn-a[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2a_activation-b (Activatio (None, 56, 56, 96)   0           block2a_bn-b[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_squeeze-a (GlobalAve (None, 96)           0           block2a_activation-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_squeeze-b (GlobalAve (None, 96)           0           block2a_activation-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reshape-a (Reshape)  (None, 1, 1, 96)     0           block2a_se_squeeze-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reshape-b (Reshape)  (None, 1, 1, 96)     0           block2a_se_squeeze-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reduce-a (Conv2D)    (None, 1, 1, 4)      388         block2a_se_reshape-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_reduce-b (Conv2D)    (None, 1, 1, 4)      388         block2a_se_reshape-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_expand-a (Conv2D)    (None, 1, 1, 96)     480         block2a_se_reduce-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_expand-b (Conv2D)    (None, 1, 1, 96)     480         block2a_se_reduce-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_excite-a (Multiply)  (None, 56, 56, 96)   0           block2a_activation-a[0][0]       \n",
      "                                                                 block2a_se_expand-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_se_excite-b (Multiply)  (None, 56, 56, 96)   0           block2a_activation-b[0][0]       \n",
      "                                                                 block2a_se_expand-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_conv-a (Conv2D) (None, 56, 56, 24)   2304        block2a_se_excite-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_conv-b (Conv2D) (None, 56, 56, 24)   2304        block2a_se_excite-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_bn-a (BatchNorm (None, 56, 56, 24)   96          block2a_project_conv-a[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2a_project_bn-b (BatchNorm (None, 56, 56, 24)   96          block2a_project_conv-b[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_conv-a (Conv2D)  (None, 56, 56, 144)  3456        block2a_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_conv-b (Conv2D)  (None, 56, 56, 144)  3456        block2a_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_bn-a (BatchNorma (None, 56, 56, 144)  576         block2b_expand_conv-a[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_bn-b (BatchNorma (None, 56, 56, 144)  576         block2b_expand_conv-b[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_activation-a (Ac (None, 56, 56, 144)  0           block2b_expand_bn-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_expand_activation-b (Ac (None, 56, 56, 144)  0           block2b_expand_bn-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_dwconv-a (DepthwiseConv (None, 56, 56, 144)  1296        block2b_expand_activation-a[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2b_dwconv-b (DepthwiseConv (None, 56, 56, 144)  1296        block2b_expand_activation-b[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "block2b_bn-a (BatchNormalizatio (None, 56, 56, 144)  576         block2b_dwconv-a[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2b_bn-b (BatchNormalizatio (None, 56, 56, 144)  576         block2b_dwconv-b[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2b_activation-a (Activatio (None, 56, 56, 144)  0           block2b_bn-a[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2b_activation-b (Activatio (None, 56, 56, 144)  0           block2b_bn-b[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_squeeze-a (GlobalAve (None, 144)          0           block2b_activation-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_squeeze-b (GlobalAve (None, 144)          0           block2b_activation-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reshape-a (Reshape)  (None, 1, 1, 144)    0           block2b_se_squeeze-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reshape-b (Reshape)  (None, 1, 1, 144)    0           block2b_se_squeeze-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reduce-a (Conv2D)    (None, 1, 1, 6)      870         block2b_se_reshape-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_reduce-b (Conv2D)    (None, 1, 1, 6)      870         block2b_se_reshape-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_expand-a (Conv2D)    (None, 1, 1, 144)    1008        block2b_se_reduce-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_expand-b (Conv2D)    (None, 1, 1, 144)    1008        block2b_se_reduce-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_excite-a (Multiply)  (None, 56, 56, 144)  0           block2b_activation-a[0][0]       \n",
      "                                                                 block2b_se_expand-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_se_excite-b (Multiply)  (None, 56, 56, 144)  0           block2b_activation-b[0][0]       \n",
      "                                                                 block2b_se_expand-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_conv-a (Conv2D) (None, 56, 56, 24)   3456        block2b_se_excite-a[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_conv-b (Conv2D) (None, 56, 56, 24)   3456        block2b_se_excite-b[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_bn-a (BatchNorm (None, 56, 56, 24)   96          block2b_project_conv-a[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_project_bn-b (BatchNorm (None, 56, 56, 24)   96          block2b_project_conv-b[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block2b_drop-a (Dropout)        (None, 56, 56, 24)   0           block2b_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_drop-b (Dropout)        (None, 56, 56, 24)   0           block2b_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_add-a (Add)             (None, 56, 56, 24)   0           block2b_drop-a[0][0]             \n",
      "                                                                 block2a_project_bn-a[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block2b_add-b (Add)             (None, 56, 56, 24)   0           block2b_drop-b[0][0]             \n",
      "                                                                 block2a_project_bn-b[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 56, 56, 24)   0           block2b_add-a[0][0]              \n",
      "                                                                 block2b_add-b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "depthwise_conv2d_4 (DepthwiseCo (None, 56, 56, 24)   48          add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 56, 56, 24)   96          depthwise_conv2d_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.selu_4 (TFOpLambda)       (None, 56, 56, 24)   0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 56, 56, 1)    25          tf.nn.selu_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "depthwise_conv2d_5 (DepthwiseCo (None, 56, 56, 1)    2           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 56, 56, 1)    4           depthwise_conv2d_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.selu_5 (TFOpLambda)       (None, 56, 56, 1)    0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 56, 56, 1)    2           tf.nn.selu_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 56, 56, 24)   0           add_4[0][0]                      \n",
      "                                                                 conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_2 (Glo (None, 24)           0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "softmaxproposed_model (Dense)   (None, 6)            150         global_average_pooling2d_2[0][0] \n",
      "==================================================================================================\n",
      "Total params: 40,953\n",
      "Trainable params: 38,457\n",
      "Non-trainable params: 2,496\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Build final model\n",
    "def model_final(models, model_input):\n",
    "    outputs = [m.output for m in models]\n",
    "    \n",
    "#Fusion\n",
    "    y = Add()(outputs)\n",
    "    \n",
    "#Residual Layer\n",
    "    y_dw1 = DepthwiseConv2D(1, 1)(y) \n",
    "    y_bn1 = BatchNormalization()(y_dw1)\n",
    "    y_selu1 = activation_setting(y_bn1)\n",
    "    y_conv1 = Conv2D(1, 1, kernel_initializer='lecun_normal')(y_selu1)\n",
    "    \n",
    "    y_dw2 = DepthwiseConv2D(1, 1)(y_conv1)\n",
    "    y_bn2 = BatchNormalization()(y_dw2)\n",
    "    y_selu2 = activation_setting(y_bn2)\n",
    "    y_conv2 = Conv2D(1, 1, kernel_initializer='lecun_normal')(y_selu2)\n",
    "    \n",
    "    y_merge = Add()([y, y_conv2])\n",
    "    \n",
    "#FINE-TUNING\n",
    "    y = GlobalAveragePooling2D()(y_merge)\n",
    "\n",
    "    prediction = Dense(6, \n",
    "                       activation='softmax', \n",
    "                       name='softmax' + model_architecture)(y)\n",
    "    \n",
    "    model_final = Model(model_input, prediction, name=model_architecture)\n",
    "    \n",
    "    return model_final\n",
    "\n",
    "#Istantitate the model and report the summary\n",
    "model_final = model_final(models, model_input)\n",
    "\n",
    "print()\n",
    "print()\n",
    "print(\"PLEASE CHECK THE MODEL UP TO THE END\")\n",
    "print()\n",
    "print()\n",
    "print()\n",
    "\n",
    "print(\"Complete and ready for compilation and training!\")\n",
    "print(\"The error is caused because of the modifications made from the original layers\")\n",
    "\n",
    "#save the architecture\n",
    "save_m(model_path + model_architecture, model_final)\n",
    "\n",
    "#re-fresh the entire architecture into the model variable for consistency\n",
    "model = load_model(model_path + model_architecture + '/' + model_architecture + '.h5')\n",
    "\n",
    "model.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}