Adding prototype unsupervised contrastive loss function with demos

.gitignore

@@ -135,4 +135,4 @@ dmypy.json
 .DS_Store
 
 #pickle
-*.pickle
+#*.pickle

benchmarks/cifar_exp/fte_bte_exp.py

@@ -1,6 +1,5 @@
 #%%
 import random
-import matplotlib.pyplot as plt
 import tensorflow as tf
 import keras
 from keras import layers
@@ -16,6 +15,8 @@
 from joblib import Parallel, delayed
 from multiprocessing import Pool
 
+from keras.optimizers import Adam
+from keras.callbacks import EarlyStopping
 from proglearn.progressive_learner import ProgressiveLearner
 from proglearn.deciders import SimpleArgmaxAverage
 from proglearn.transformers import NeuralClassificationTransformer, TreeClassificationTransformer
@@ -24,13 +25,33 @@
 import tensorflow as tf
 
 import time
-
+import sys
 #%%
 def unpickle(file):
     with open(file, 'rb') as fo:
         dict = pickle.load(fo, encoding='bytes')
     return dict
 
+def get_size(obj, seen=None):
+    """Recursively finds size of objects"""
+    size = sys.getsizeof(obj)
+    if seen is None:
+        seen = set()
+    obj_id = id(obj)
+    if obj_id in seen:
+        return 0
+    # Important mark as seen *before* entering recursion to gracefully handle
+    # self-referential objects
+    seen.add(obj_id)
+    if isinstance(obj, dict):
+        size += sum([get_size(v, seen) for v in obj.values()])
+        size += sum([get_size(k, seen) for k in obj.keys()])
+    elif hasattr(obj, '__dict__'):
+        size += get_size(obj.__dict__, seen)
+    '''elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)):
+        size += sum([get_size(i, seen) for i in obj])'''
+    return size
+
 #%%
 def LF_experiment(train_x, train_y, test_x, test_y, ntrees, shift, slot, model, num_points_per_task, acorn=None):
 
@@ -43,6 +64,8 @@ def LF_experiment(train_x, train_y, test_x, test_y, ntrees, shift, slot, model,
     train_times_across_tasks = []
     single_task_inference_times_across_tasks = []
     multitask_inference_times_across_tasks = []
+    time_info = []
+    mem_info = []
 
     if model == "dnn":
         default_transformer_class = NeuralClassificationTransformer
@@ -66,12 +89,19 @@ def LF_experiment(train_x, train_y, test_x, test_y, ntrees, shift, slot, model,
         network.add(layers.BatchNormalization())
         network.add(layers.Dense(units=10, activation = 'softmax'))
 
-        default_transformer_kwargs = {"network" : network,
-                                      "euclidean_layer_idx" : -2,
-                                      "num_classes" : 10,
-                                      "optimizer" : keras.optimizers.Adam(3e-4)
-                                     }
-
+        default_transformer_kwargs = {
+            "network": network,
+            "euclidean_layer_idx": -2,
+            "loss": "categorical_crossentropy",
+            "optimizer": Adam(3e-4),
+            "fit_kwargs": {
+                "epochs": 100,
+                "callbacks": [EarlyStopping(patience=5, monitor="val_loss")],
+                "verbose": False,
+                "validation_split": 0.33,
+                "batch_size": 32,
+            },
+        }
         default_voter_class = KNNClassificationVoter
         default_voter_kwargs = {"k" : int(np.log2(num_points_per_task))}
 
@@ -94,28 +124,50 @@ def LF_experiment(train_x, train_y, test_x, test_y, ntrees, shift, slot, model,
 
     for task_ii in range(10):
         print("Starting Task {} For Fold {}".format(task_ii, shift))
+
+
+        train_start_time = time.time()
+
         if acorn is not None:
             np.random.seed(acorn)
 
-        train_start_time = time.time()
         progressive_learner.add_task(
             X = train_x[task_ii*5000+slot*num_points_per_task:task_ii*5000+(slot+1)*num_points_per_task],
             y = train_y[task_ii*5000+slot*num_points_per_task:task_ii*5000+(slot+1)*num_points_per_task],
             num_transformers = 1 if model == "dnn" else ntrees,
-            transformer_voter_decider_split = [0.67, 0.33, 0],
+            transformer_voter_decider_split = [0.63, 0.37, 0],
             decider_kwargs = {"classes" : np.unique(train_y[task_ii*5000+slot*num_points_per_task:task_ii*5000+(slot+1)*num_points_per_task])}
             )
         train_end_time = time.time()
+
+        single_learner = ProgressiveLearner(default_transformer_class = default_transformer_class,
+                                         default_transformer_kwargs = default_transformer_kwargs,
+                                         default_voter_class = default_voter_class,
+                                         default_voter_kwargs = default_voter_kwargs,
+                                         default_decider_class = default_decider_class)
+
+        if acorn is not None:
+            np.random.seed(acorn)
+
+        single_learner.add_task(
+            X = train_x[task_ii*5000+slot*num_points_per_task:task_ii*5000+(slot+1)*num_points_per_task],
+            y = train_y[task_ii*5000+slot*num_points_per_task:task_ii*5000+(slot+1)*num_points_per_task],
+            num_transformers = 1 if model == "dnn" else (task_ii+1)*ntrees,
+            transformer_voter_decider_split = [0.67, 0.33, 0],
+            decider_kwargs = {"classes" : np.unique(train_y[task_ii*5000+slot*num_points_per_task:task_ii*5000+(slot+1)*num_points_per_task])}
+            )
 
+        time_info.append(train_end_time - train_start_time)
+        mem_info.append(get_size(progressive_learner))
         train_times_across_tasks.append(train_end_time - train_start_time)
 
         single_task_inference_start_time = time.time()
-        llf_task=progressive_learner.predict(
-            X = test_x[task_ii*1000:(task_ii+1)*1000,:], transformer_ids=[task_ii], task_id=task_ii
+        single_task=single_learner.predict(
+            X = test_x[task_ii*1000:(task_ii+1)*1000,:], transformer_ids=[0], task_id=0
             )
         single_task_inference_end_time = time.time()
         single_task_accuracies[task_ii] = np.mean(
-                llf_task == test_y[task_ii*1000:(task_ii+1)*1000]
+                single_task == test_y[task_ii*1000:(task_ii+1)*1000]
                     )
         single_task_inference_times_across_tasks.append(single_task_inference_end_time - single_task_inference_start_time)
 
@@ -149,11 +201,20 @@ def LF_experiment(train_x, train_y, test_x, test_y, ntrees, shift, slot, model,
     df_single_task['single_task_inference_times'] = single_task_inference_times_across_tasks
     df_single_task['train_times'] = train_times_across_tasks
 
+    #print(df)
     summary = (df,df_single_task)
-    file_to_save = 'result/result/'+model+str(ntrees)+'_'+str(shift)+'_'+str(slot)+'.pickle'
+    file_to_save = 'result/result/'+model+str(ntrees)+'_'+str(shift)+'.pickle'
     with open(file_to_save, 'wb') as f:
         pickle.dump(summary, f)
 
+    '''file_to_save = 'result/time_res/'+model+str(ntrees)+'_'+str(shift)+'_'+str(slot)+'.pickle'
+    with open(file_to_save, 'wb') as f:
+        pickle.dump(time_info, f)
+
+    file_to_save = 'result/mem_res/'+model+str(ntrees)+'_'+str(shift)+'_'+str(slot)+'.pickle'
+    with open(file_to_save, 'wb') as f:
+        pickle.dump(mem_info, f)'''
+
 #%%
 def cross_val_data(data_x, data_y, num_points_per_task, total_task=10, shift=1):
     x = data_x.copy()
@@ -187,9 +248,9 @@ def run_parallel_exp(data_x, data_y, n_trees, model, num_points_per_task, slot=0
     train_x, train_y, test_x, test_y = cross_val_data(data_x, data_y, num_points_per_task, shift=shift)
 
     if model == "dnn":
-        config = tf.ConfigProto()
+        config = tf.compat.v1.ConfigProto()
         config.gpu_options.allow_growth = True
-        sess = tf.Session(config=config)
+        sess = tf.compat.v1.Session(config=config)
         with tf.device('/gpu:'+str(shift % 4)):
             LF_experiment(train_x, train_y, test_x, test_y, n_trees, shift, slot, model, num_points_per_task, acorn=12345)
     else:
@@ -198,7 +259,7 @@ def run_parallel_exp(data_x, data_y, n_trees, model, num_points_per_task, slot=0
 #%%
 ### MAIN HYPERPARAMS ###
 model = "uf"
-num_points_per_task = 500
+num_points_per_task = 5000
 ########################
 
 (X_train, y_train), (X_test, y_test) = keras.datasets.cifar100.load_data()
@@ -211,7 +272,7 @@ def run_parallel_exp(data_x, data_y, n_trees, model, num_points_per_task, slot=0
 
 #%%
 if model == "uf":
-    slot_fold = range(10)
+    slot_fold = range(1)
     shift_fold = range(1,7,1)
     n_trees=[10]
     iterable = product(n_trees,shift_fold,slot_fold)
@@ -239,4 +300,12 @@ def perform_shift(shift_slot_tuple):
     with Pool(4) as p:
         p.map(perform_shift, stage_2_iterable)
 
-# %%
+'''slot_fold = range(1)
+shift_fold = [1,2,3,4,5,6]
+n_trees=[0]
+iterable = product(n_trees,shift_fold,slot_fold)
+
+for ntree,shift,slot in iterable:
+    run_parallel_exp(
+                data_x, data_y, ntree, model, num_points_per_task, slot=slot, shift=shift
+                )'''