Add experimental compilation options tuners in Python

python/experimental/options_search/eval_genetic_options.py

@@ -0,0 +1,23 @@
+import tensor_comprehensions as tc
+import tensor_comprehensions.tclib as tclib
+import utils
+
+cache = tc.MappingOptionsCache("genetic_savedopt_conv_default.txt")
+
+exptuner_config = utils.ExpTunerConfig()
+exptuner_config.set_convolution_tc()
+tc_code, tc_name, inp = exptuner_config.tc_code, exptuner_config.tc_name, exptuner_config.inp
+
+print("divs : " + str(utils.getAllDivs(inp)))
+tup = cache.load(tc_code, tc_name, inp, 1)
+if(tup == []):
+    exit()
+best_options, = tup
+best_options = best_options.getDict()
+optsVect = utils.getRawVectorFromTcOpt(best_options)
+opts = utils.optionsFromVector(optsVect)
+print(opts)
+
+time = utils.evalTime(opts, exptuner_config, estimator="median")
+print(time)
+

python/experimental/options_search/eval_options.py

@@ -0,0 +1,21 @@
+import numpy as np
+import time
+import utils
+import torch
+import torch.optim as optim
+import torch.nn as nn
+import torch.utils.data
+import torch.nn.functional as F
+import tensor_comprehensions as tc
+
+set_options = [
+[1, 1, 0, 1, 2, 1, 3, 6, 8, 0, 3, 0, 2, 11, 9, 8, 2, 0, 0, 3, 0, 0, 1, 0, 1, 2]
+]
+
+exptuner_config = utils.ExpTunerConfig()
+exptuner_config.set_convolution_tc()
+
+for i in range(len(set_options)):
+    opts = np.array(set_options[i])
+    time = utils.evalTime(opts, exptuner_config)
+    print(time)

python/experimental/options_search/generate_genetic_options.py

@@ -0,0 +1,16 @@
+import numpy as np
+import torch
+import tensor_comprehensions as tc
+
+import utils
+
+exptuner_config = utils.ExpTunerConfig()
+exptuner_config.set_convolution_tc()
+tc_code, tc_name, inp = exptuner_config.tc_code, exptuner_config.tc_name, exptuner_config.inp
+#config = tc.autotuner_settings
+#config["pop_size"]=50
+#config["generations"]=1
+opts = tc.MappingOptions("naive")
+print(opts)
+
+tc.autotune(tc_code, tc_name, *inp, starting_options=opts, cache_filename="genetic_savedopt_conv_default.txt", store_to_cache=True)

python/experimental/options_search/mcts.py

@@ -0,0 +1,179 @@
+import tensor_comprehensions as tc
+import torch
+import utils
+import numpy as np
+#from tqdm import tqdm
+from visdom import Visdom
+
+viz = Visdom()
+
+class Node:
+    def __init__(self, father=None, new_act=0):
+        self.value = 0
+        self.values = []
+        self.nbVisits=0
+        self.nbChildrenSeen = 0
+        self.pos=0
+        #self.hasSeen = {} #todo
+        self.children=[]
+        self.parent = father
+        self.stateVector = [0] * utils.NB_HYPERPARAMS
+        if(father != None):
+            self.pos = father.pos+1
+            #self.hasSeen = {} #todo
+            self.stateVector = father.stateVector[:]
+            self.stateVector[self.pos-1] = new_act
+
+    def getRoot(self):
+        return self
+
+    def getParent(self):
+        return self.parent
+
+    def notRoot(self):
+        return (self.parent != None)
+
+class MCTS:
+    def __init__(self):
+        self.C = 1 #to tune
+
+        self.exptuner_config = utils.ExpTunerConfig()
+        self.exptuner_config.set_convolution_tc()
+
+        self.nbActions = self.exptuner_config.cat_sz
+        self.tree = Node()
+
+        self.best_rewards = []
+        self.rws = []
+
+        self.curIter=0
+        self.curr_best=0
+        self.running_reward=0
+        self.win0 = viz.line(X=np.arange(5), Y=np.random.rand(5))
+
+    def main_search(self, starting_pos): #, init_inp):
+        node = starting_pos
+        #node.nbVisits+=1
+        ttNbIters = 10 #2*self.nbActions[node.pos]
+        for _ in range(max(ttNbIters, self.nbActions[node.pos])):
+            leaf = self.getLeaf(node)
+            val = self.evaluate(leaf)
+            self.backup(leaf, val)
+            #print(node.value / node.nbVisits)
+        _, action = self.getBestChild2(node)
+        return action
+
+    def take_action(self, node, act):
+        if(node.nbChildrenSeen > act):
+            return node.children[act]
+        new_child = Node(father=node, new_act=act)
+        node.children.append(new_child)
+        #node.hasSeen[act]=1
+        node.nbChildrenSeen += 1
+        return node.children[-1]
+
+    def getLeaf(self, node):
+        first=True
+        while(node.pos < utils.NB_HYPERPARAMS and (first or node.nbVisits != 0)):
+            first=False
+            pos = node.pos
+            if(node.nbChildrenSeen == self.nbActions[pos]):
+                node, _ = self.getBestChild(node)
+            else:
+                act=node.nbChildrenSeen
+                self.take_action(node, act)
+                return node.children[-1]
+        return node
+
+    def getBestChild2(self, node):
+        bestIndic = 0.
+        bestAction = 0
+        first=True
+        pos = node.pos
+        for act in range(self.nbActions[pos]):
+            child = node.children[act]
+            #indic = np.percentile(child.values, 20)
+            indic = child.value / child.nbVisits
+            if(first or indic > bestIndic):
+                bestIndic = indic
+                bestAction = act
+                first=False
+        return node.children[bestAction], bestAction
+
+    def getBestChild(self, node):
+        bestIndic = 0.
+        bestAction = 0
+        first=True
+        pos = node.pos
+        for act in range(self.nbActions[pos]):
+            child = node.children[act]
+            #indic = np.percentile(child.values, 20) + self.C * np.sqrt(2*np.log(node.nbVisits) / child.nbVisits)
+            indic = child.value / child.nbVisits + self.C * np.sqrt(2*np.log(node.nbVisits) / child.nbVisits)
+            if(first or indic > bestIndic):
+                bestIndic = indic
+                bestAction = act
+                first=False
+        return node.children[bestAction], bestAction
+
+    def saveReward(self, reward, opts):
+        INTER_DISP = 20
+        #print(-reward)
+        if(self.curIter == 0):
+            self.running_reward = reward
+            self.curr_best = reward
+        if(self.curIter == 0 or reward > self.curr_best):
+            print(-reward)
+            print(opts)
+        self.curIter += 1
+        self.running_reward = self.running_reward * 0.99 + reward * 0.01
+        self.curr_best = max(self.curr_best, reward)
+        #self.rewards.append(-reward)
+        self.best_rewards.append(-self.curr_best)
+        self.rws.append(-self.running_reward)
+        if self.curIter % INTER_DISP == 0:
+            viz.line(X=np.column_stack((np.arange(self.curIter), np.arange(self.curIter))), \
+            Y=np.column_stack((np.array(self.rws), np.array(self.best_rewards))), \
+            win=self.win0, opts=dict(legend=["Geometric run", "Best time"]))
+
+    def randomSampleScoreFrom(self, node):
+        pos = node.pos
+        optsVector = node.stateVector
+        for i in range(utils.NB_HYPERPARAMS - (pos)):
+            a = np.random.randint(self.nbActions[i+pos])
+            optsVector[i+(pos)] = a
+        #print(optsVector)
+        reward = -np.log(utils.evalTime(optsVector, self.exptuner_config))
+        self.saveReward(reward, optsVector)
+        return reward
+
+    def evaluate(self, leaf):
+        score = 0
+        nb_iters=5
+        for _ in range(nb_iters):
+            score += self.randomSampleScoreFrom(leaf)
+        return score / nb_iters
+
+    def backup(self, leaf, val):
+        #if(val > 10.): #infty
+        #    return
+        node = leaf
+        while(node.notRoot()):
+            node.nbVisits += 1
+            #node.values.append(val)
+            node.value += val
+            node = node.getParent()
+        node.nbVisits += 1
+        node.value += val
+        node.values.append(val)
+
+mcts = MCTS()
+
+opts = []
+curr_node = mcts.tree
+for i in range(utils.NB_HYPERPARAMS):
+    opts.append(mcts.main_search(curr_node))
+    curr_node = mcts.take_action(curr_node, opts[-1])
+    print(opts)
+opts = np.array(opts).astype(int)
+print(utils.evalTime(opts.tolist(), mcts.exptuner_config))
+utils.print_opt(opts)

python/experimental/options_search/predict_time.py

@@ -0,0 +1,57 @@
+import time
+import torch
+import tensor_comprehensions as tc
+#import sklearn
+#from sklearn.linear_model import LinearRegression
+#from sklearn.ensemble import GradientBoostingRegressor
+import numpy as np
+#from sklearn.model_selection import train_test_split
+#from tensor_comprehensions.mapping_options import Options
+from multiprocessing import Pool
+from itertools import repeat
+import utils
+#from tqdm import tqdm
+
+exptuner_config = utils.ExpTunerConfig()
+exptuner_config.set_convolution_tc()
+
+NB_HYPERPARAMS = utils.NB_HYPERPARAMS
+
+def createY(x):
+    y = utils.evalTime(x, exptuner_config)
+    return y
+
+def getRandom():
+    opt_v = np.zeros(NB_HYPERPARAMS).astype(int)
+    for i in range(opt_v.shape[0]):
+        opt_v[i] = np.random.randint(exptuner_config.cat_sz[i])
+    return opt_v
+
+def makeDataset():
+    from tqdm import tqdm
+    sz = 500
+    datasetX, datasetY = [], []
+    for _ in tqdm(range(sz)):
+        opt = getRandom()
+        yi = createY(opt)
+        datasetX.append(opt)
+        datasetY.append(yi)
+    #with Pool(sz) as p:
+    #    datasetY = p.starmap(createY, datasetX)
+    return np.array(datasetX), np.array(datasetY)
+
+def learn():
+    #from sklearn.linear_model import LinearRegression
+    from sklearn.ensemble import GradientBoostingRegressor
+    from sklearn.model_selection import train_test_split
+    datasetX, datasetY = makeDataset()
+    print(min(datasetY))
+    Xtrain, Xtest, Ytrain, Ytest = train_test_split(datasetX, datasetY, test_size=0.2, random_state = 42)
+    model1 = GradientBoostingRegressor(n_estimators=1000)
+    model1.fit(Xtrain, Ytrain)
+    pred0 = model1.predict(Xtrain)
+    pred1 = model1.predict(Xtest)
+    print(np.corrcoef(pred0, Ytrain)[0, 1]**2)
+    print(np.corrcoef(pred1, Ytest)[0,1]**2)
+
+#learn()

python/experimental/options_search/random_search.py

@@ -0,0 +1,63 @@
+import numpy as np
+#import ipdb
+import torch
+import torch.optim as optim
+import torch.nn as nn
+import torch.utils.data
+import torch.nn.functional as F
+import tensor_comprehensions as tc
+from visdom import Visdom
+
+import utils
+
+NB_EPOCHS = 1000
+BATCH_SZ = 1
+
+viz = Visdom()
+win0 = viz.line(X=np.arange(NB_EPOCHS), Y=np.random.rand(NB_EPOCHS))
+
+exptuner_config = utils.ExpTunerConfig()
+exptuner_config.set_convolution_tc()
+
+NB_HYPERPARAMS = utils.NB_HYPERPARAMS
+
+def getRandom():
+    opt_v = np.zeros(NB_HYPERPARAMS).astype(int)
+    for i in range(opt_v.shape[0]):
+        opt_v[i] = np.random.randint(exptuner_config.cat_sz[i])
+    return opt_v
+
+INTER_DISP = 20
+
+running_reward = -0.5
+tab_rewards=[]
+tab_best=[]
+best=-12
+best_options = -1
+for i in range(NB_EPOCHS):
+    rewards = []
+    opts=[]
+    for j in range(BATCH_SZ):
+        out = getRandom()
+        reward = utils.evalTime(out.astype(int), exptuner_config, prune=2, curr_best=np.exp(-best))
+        reward = -np.log(reward)
+        rewards.append(reward)
+        opts.append(out.astype(int))
+    if(best < np.max(rewards) or i==0):
+        best = np.max(rewards)
+        ind=np.argmax(rewards)
+        best_options = opts[ind]
+        utils.print_opt(best_options)
+    if(i==0):
+        running_reward = reward
+    running_reward = running_reward * 0.99 + np.mean(rewards) * 0.01
+    tab_rewards.append(-running_reward)
+    tab_best.append(-best)
+    if i % INTER_DISP == 0:
+        viz.line(X=np.column_stack((np.arange(i+1), np.arange(i+1))), Y=np.column_stack((np.array(tab_rewards), np.array(tab_best))), win=win0, opts=dict(legend=["Geometric run", "Best time"]))
+    print(-running_reward)
+    print(-best)
+tab_best = np.array(tab_best)
+np.save("randomsearch.npy", tab_best)
+print("Finally, best options are:")
+utils.print_opt(best_options)