DeepLearningLab/nb201.py at main · donats11/DeepLearningLab · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
import json
import pickle
import numpy as np
import ConfigSpace as CS

class NB201Benchmark(object):
    def __init__(self, path, dataset='cifar10', seed=None):
        cs = self.get_configuration_space()
        self.names = [h.name for h in cs.get_hyperparameters()]
        self.data = self._load_data()
        self.dataset = dataset

        self.X = []
        self.y = []
        self.c = []

        self.rng = np.random.RandomState(seed)

    def _load_data(self):
        with open('nb201.pkl', 'rb') as f:
            data = pickle.load(f)
        return data

    def reset_tracker(self, seed):
        # __init__() sans the data loading for multiple runs
        self.X = []
        self.y = []
        self.c = []
        self.rng = np.random.RandomState(seed)

    @staticmethod
    def get_configuration_space():
        cs = CS.ConfigurationSpace()
        # op_i_to_j denotes the edge that connects node i to j in the DAG representation of the NB201 search space
        cs.add_hyperparameter(CS.CategoricalHyperparameter("op_0_to_1", ["none", "skip_connect", "avg_pool_3x3", "nor_conv_1x1", "nor_conv_3x3"]))
        cs.add_hyperparameter(CS.CategoricalHyperparameter("op_0_to_2", ["none", "skip_connect", "avg_pool_3x3", "nor_conv_1x1", "nor_conv_3x3"]))
        cs.add_hyperparameter(CS.CategoricalHyperparameter("op_1_to_2", ["none", "skip_connect", "avg_pool_3x3", "nor_conv_1x1", "nor_conv_3x3"]))
        cs.add_hyperparameter(CS.CategoricalHyperparameter("op_0_to_3", ["none", "skip_connect", "avg_pool_3x3", "nor_conv_1x1", "nor_conv_3x3"]))
        cs.add_hyperparameter(CS.CategoricalHyperparameter("op_1_to_3", ["none", "skip_connect", "avg_pool_3x3", "nor_conv_1x1", "nor_conv_3x3"]))
        cs.add_hyperparameter(CS.CategoricalHyperparameter("op_2_to_3", ["none", "skip_connect", "avg_pool_3x3", "nor_conv_1x1", "nor_conv_3x3"]))
        return cs

    def get_best_configuration(self):

        """
        Returns the best configuration in the dataset that achieves the lowest test performance.

        :return: Returns tuple with the best configuration, its final validation performance and its test performance
        """

        configs, te, ve = [], [], []
        configs = list(sorted(self.data, key=lambda x: self.data[x][self.dataset]))
        best_config = configs[-1]
        best_test_error = 100 - self.data[best_config][self.dataset]

        return best_config, best_test_error

    def objective_function(self, config, **kwargs):
        if type(config) == CS.Configuration:
            k = json.dumps(config.get_dictionary(), sort_keys=True)
        else:
            k = json.dumps(config, sort_keys=True)

        # convert the CS representation to the nb201 string representation
        nb201_arch = "|{}~0|+|{}~0|{}~1|+|{}~0|{}~1|{}~2|".format(*config.get_dictionary().values())

        error = 100 - self.data[nb201_arch][self.dataset]

        # latency in ms with a batch size of 32 on a GTX 1080Ti
        time_per_minibatch = self.data[nb201_arch]['1080ti_32_latency']
        time_per_epoch = time_per_minibatch * 50000/32 # 50000 images in CIFAR10
        total_runtime = 200 * time_per_epoch / 1000 # in seconds; 200 epochs in total

        self.X.append(config)
        self.y.append(error)
        self.c.append(total_runtime)

        return error, total_runtime