rbf-fine-tuning_grid.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport tensorflow as tf\nfrom keras.layers import Dense, Dropout\nfrom keras.models import Sequential\nfrom keras.datasets import boston_housing\nfrom keras import layers\nfrom keras.initializers import Initializer\nfrom sklearn.cluster import KMeans\nimport time\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:42.231327Z","iopub.execute_input":"2022-09-19T13:11:42.231794Z","iopub.status.idle":"2022-09-19T13:11:50.335051Z","shell.execute_reply.started":"2022-09-19T13:11:42.231739Z","shell.execute_reply":"2022-09-19T13:11:50.333565Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":"# Definition of the RBF Layer\nThe code is based on the implementation of Petra Vidnerova et. all, which can be found [here](https://github.com/PetraVidnerova/rbf_keras). Also, the [Keras Tutorial on creating a custom layer](https://www.tutorialspoint.com/keras/keras_customized_layer.htm) is followed.","metadata":{}},{"cell_type":"code","source":"class RBFLayer(layers.Layer):\n    # output_dim: number of hidden units (number of outputs of the layer)\n    # initializer: instance of initializer to initialize centers\n    # betas: float, initial value for betas (beta = 1 / 2*pow(sigma,2))\n    def __init__(self, output_dim, initializer, betas=1.0, **kwargs):\n        self.betas = betas\n        self.output_dim = output_dim\n        self.initializer = initializer\n        super(RBFLayer, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.centers = self.add_weight(name='centers', shape=(self.output_dim, input_shape[1]),\n                                       initializer=self.initializer, trainable=False)\n        sigma = np.zeros(self.output_dim)\n        for i in range(0, self.output_dim):\n            d_max = 0\n            for j in range(0, self.output_dim):\n                d = np.linalg.norm(self.centers[i] - self.centers[j])\n                if d > d_max:\n                    d_max = d\n            sigma[i] = d_max / np.sqrt(2 * self.output_dim)\n        self.betas = np.ones(self.output_dim) / (2 * (sigma ** 2))\n        super(RBFLayer, self).build(input_shape)\n\n    def call(self, inputs, *args, **kwargs):\n        C = tf.expand_dims(self.centers, -1)  \n        H = tf.transpose(C - tf.transpose(inputs))  \n        return tf.exp(-self.betas * tf.math.reduce_sum(H ** 2, axis=1))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.output_dim","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:50.337427Z","iopub.execute_input":"2022-09-19T13:11:50.338260Z","iopub.status.idle":"2022-09-19T13:11:50.352860Z","shell.execute_reply.started":"2022-09-19T13:11:50.338209Z","shell.execute_reply":"2022-09-19T13:11:50.351402Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"## KMeans Definition for the Centers\nPetra Vinderova also implements the [KMeans Initialiser](https://github.com/PetraVidnerova/rbf_keras/blob/master/kmeans_initializer.py)","metadata":{}},{"cell_type":"code","source":"class InitCentersKMeans(Initializer):\n    def __init__(self, X, max_iter=100):\n        self.X = X\n        self.max_iter = max_iter\n        super().__init__()\n\n    def __call__(self, shape, dtype=None, *args):\n        assert shape[1] == self.X.shape[1]\n        n_centers = shape[0]\n        km = KMeans(n_clusters=n_centers, max_iter=self.max_iter, verbose=0)\n        km.fit(self.X)\n        return km.cluster_centers_\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:50.354464Z","iopub.execute_input":"2022-09-19T13:11:50.355210Z","iopub.status.idle":"2022-09-19T13:11:50.375529Z","shell.execute_reply.started":"2022-09-19T13:11:50.355167Z","shell.execute_reply":"2022-09-19T13:11:50.374125Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":"## Definition of the necessary metrics\nThe following metrics are based on these implementations:\n- [R2 score](https://jmlb.github.io/ml/2017/03/20/CoeffDetermination_CustomMetric4Keras/)\n- [Root Mean Squared Error](https://stackoverflow.com/questions/43855162/rmse-rmsle-loss-function-in-keras) / and this [issue](https://github.com/keras-team/keras/issues/10706)\n- [Mean Squared Error](https://stackoverflow.com/questions/67216102/keras-mean-squared-error-mse-calculation-definition-for-images)","metadata":{}},{"cell_type":"code","source":"def root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true)))\n\n\ndef mean_squared_error(y_true, y_pred):\n    return K.mean(K.square(y_pred - y_true))\n\n\ndef r2_score(y_true, y_pred):\n    SS_res = K.sum(K.square(y_true - y_pred))\n    SS_tot = K.sum(K.square(y_true - K.mean(y_true)))\n    return 1 - SS_res/(SS_tot + K.epsilon())","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:50.379011Z","iopub.execute_input":"2022-09-19T13:11:50.380006Z","iopub.status.idle":"2022-09-19T13:11:50.388813Z","shell.execute_reply.started":"2022-09-19T13:11:50.379953Z","shell.execute_reply":"2022-09-19T13:11:50.387748Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"markdown","source":"## Other Functions that will be used","metadata":{}},{"cell_type":"code","source":"def data_normalisation(data):\n    encoder = StandardScaler()\n    return encoder.fit_transform(data)\n\ndef split_validation(data, length_training, length_validation):\n    data_train, data_validation = tf.split(data, [length_training, length_validation], 0)\n    return data_train, data_validation","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:50.390405Z","iopub.execute_input":"2022-09-19T13:11:50.391406Z","iopub.status.idle":"2022-09-19T13:11:50.407852Z","shell.execute_reply.started":"2022-09-19T13:11:50.391367Z","shell.execute_reply":"2022-09-19T13:11:50.406597Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{}},{"cell_type":"code","source":"test_length = 0.25\n(x_train, y_train), (x_test, y_test) = boston_housing.load_data(test_split=test_length)\n\nnormalise = True\nif normalise:\n    x_train = data_normalisation(x_train)\n    x_test = data_normalisation(x_test)\n    \n# length_validation = round(0.2 * len(x_train))\n# length_training = round(0.8 * len(x_train))\n# x_train, x_validation = split_validation(x_train, length_training, length_validation)\n# y_train, y_validation = split_validation(y_train, length_training, length_validation)\n\n# x = tf.concat([x_train, x_validation], 0).numpy()\n# y = tf.concat([y_train, y_validation], 0).numpy()\n# print(len(x))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:54:25.438069Z","iopub.status.idle":"2022-09-19T13:54:25.438447Z","shell.execute_reply.started":"2022-09-19T13:54:25.438263Z","shell.execute_reply":"2022-09-19T13:54:25.438281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters","metadata":{}},{"cell_type":"code","source":"times = [] \nk = 5  # 5-fold cross validation\nlearning_rate = 0.001\nepochs = 100\nbatch_size = 256\n\nneurons_percentage = [0.05, 0.15, 0.3, 0.5]\ntotal_neurons_rbf = [round(n*0.8*x_train.shape[0]) for n in neurons_percentage]\ntotal_neurons_hidden2 = [32, 64, 128, 256]\ntotal_dropout_probabilities = [0.2, 0.35, 0.5]\nneurons_output = 1\n\ngrid = []\nindex = 0\nstart = time.time()\nrmse_final = []","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:50.540296Z","iopub.execute_input":"2022-09-19T13:11:50.540661Z","iopub.status.idle":"2022-09-19T13:11:50.549065Z","shell.execute_reply.started":"2022-09-19T13:11:50.540606Z","shell.execute_reply":"2022-09-19T13:11:50.547722Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"markdown","source":"## Define the baseline model","metadata":{}},{"cell_type":"code","source":"def baseline_model(x_train, y_train, x_val, y_val, neurons_rbf=128, neurons_h2=128, neurons_output=1, drop_prob=0, batch_size=256, epochs=100):\n    model = Sequential()\n    model.add(RBFLayer(neurons_rbf, initializer=InitCentersKMeans(x_train), input_shape=(13, )))\n    model.add(Dense(neurons_h2))\n    model.add(Dropout(drop_prob))\n    model.add(Dense(neurons_output))\n\n    model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate), loss=mean_squared_error,metrics=[r2_score, root_mean_squared_error])\n    fit_model = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_val, y_val),verbose=0)\n    return fit_model, model\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:11:50.550852Z","iopub.execute_input":"2022-09-19T13:11:50.551292Z","iopub.status.idle":"2022-09-19T13:11:50.562311Z","shell.execute_reply.started":"2022-09-19T13:11:50.551232Z","shell.execute_reply":"2022-09-19T13:11:50.560899Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"markdown","source":"## Fine Tuning","metadata":{}},{"cell_type":"code","source":"for dropout_probability in total_dropout_probabilities:\n    for neurons_hidden2 in total_neurons_hidden2:\n        n = 0\n        for neurons_rbf in total_neurons_rbf:\n            rmse = []\n            kf = KFold(k, shuffle=True, random_state=42)\n            for train, test in kf.split(x_train):\n                x_fold_train = x_train[train]\n                y_fold_train = y_train[train]\n                x_fold_validation = x_train[test]\n                y_fold_validation = y_train[test]\n                \n                fit_model, model = baseline_model(x_fold_train, y_fold_train, x_fold_validation, y_fold_validation, neurons_rbf, neurons_hidden2, neurons_output,\n                                          dropout_probability, batch_size, epochs)\n                rmse.append(min(fit_model.history['root_mean_squared_error']))\n\n            rmse_final.append(np.mean(rmse))\n            grid.append((neurons_percentage[n], dropout_probability, neurons_hidden2, neurons_rbf))\n            n +=1","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:54:25.436679Z","iopub.status.idle":"2022-09-19T13:54:25.437078Z","shell.execute_reply.started":"2022-09-19T13:54:25.436881Z","shell.execute_reply":"2022-09-19T13:54:25.436900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Find the best result","metadata":{}},{"cell_type":"code","source":"# times.append((time.time()-start))\n# print(f'Time needed: '+str(times[0])\nx = 3\nprint(x)\n# min_value_of_loss_function = min(rmse_final)\n# index_of_min = [i for i, value in enumerate(rmse_final) if value == min_value_of_loss_function]\n# min_rmse = rmse_final[index_of_min]\n# best_rbf_neuron_per = grid[index_of_min][0]\n# best_drop_prob = grid[index_of_min][1]      \n# best_h2_neuron = grid[index_of_min][2]  \n# best_rbf_neuron = grid[index_of_min][3]\n# print(f'Min RMSE: '+str(rmse_final[i]))\n# print(f'Percentage of neurons in RBF Layer: '+str(100*best_rbf_neuron_per)+'%')\n# print(f'Number of neurons in RBF Layer: '+str(best_rbf_neuron))\n# print(f'Number of neurons in second hidden Layer: '+str(best_h2_neuron))\n# print(f'Dropout Propability: '+str(best_drop_prob))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:54:25.435067Z","iopub.status.idle":"2022-09-19T13:54:25.435461Z","shell.execute_reply.started":"2022-09-19T13:54:25.435273Z","shell.execute_reply":"2022-09-19T13:54:25.435292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run the optimised model","metadata":{}},{"cell_type":"code","source":"length_validation = round(0.2 * len(x_train))\nlength_training = round(0.8 * len(x_train))\nx_train, x_validation = split_validation(x_train, length_training, length_validation)\ny_train, y_validation = split_validation(y_train, length_training, length_validation)\n\n\nfit_model, model = baseline_model(x_train, y_train, x_validation, y_validation, best_rbf_neuron, best_h2_neuron, neurons_output, \n                          best_drop_prob)\nscore = model.evaluate(x_test, y_test)\n\nmodel.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.001), loss=mean_squared_error,\n              metrics=[r2_score, root_mean_squared_error])\nhistory = model.fit(Trnx, Trny, batch_size=4, epochs=100, validation_data=(valx, valy))\nscore = model.evaluate(Tstx, Tsty)\n\ntrain_loss = [fit_model.history['loss'][i] for i in range(100)]\nvalidation_loss = [fit_model.history['val_loss'][i] for i in range(100)]\n\ntrain_r2 = fit_model.history['r2_score']\nvalidation_r2 = fit_model.history['val_r2_score']\n\ntrain_rmse = fit_model.history['root_mean_squared_error']\nvalidation_rmse = fit_model.history['val_root_mean_squared_error']\n\nplt.plot(train_loss)\nplt.plot(validation_loss)\n\nplt.title('Learning Curve')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend(['Training Loss', 'Validation Loss'], loc='upper right')\nplt.grid()\nplt.show()\n\nplt.plot(train_r2)\nplt.plot(validation_r2)\n\nplt.title('R2')\nplt.xlabel('Epochs')\nplt.ylabel('R2')\nplt.legend(['Training R2', 'Validation R2'], loc='lower right')\nplt.grid()\nplt.show()\n\nplt.plot(train_rmse)\nplt.plot(validation_rmse)\n\nplt.title('RMSE')\nplt.xlabel('Epochs')\nplt.ylabel('RMSE')\nplt.legend(['Training RMSE', 'Validation RMSE'], loc='lower right')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T13:47:52.661597Z","iopub.status.idle":"2022-09-19T13:47:52.662511Z","shell.execute_reply.started":"2022-09-19T13:47:52.662189Z","shell.execute_reply":"2022-09-19T13:47:52.662220Z"},"trusted":true},"execution_count":null,"outputs":[]}]}