Tests consomation memoire/temps + methode KL divergence (UDA)

higher/compare_res.py

@@ -6,7 +6,9 @@ if __name__ == "__main__":
     files=[
         #"res/good_TF_tests/log/Aug_mod(Data_augV5(Mix0.5-14TFx2-MagFxSh)-LeNet)-100 epochs (dataug:0)- 0 in_it.json",
         #"res/good_TF_tests/log/Aug_mod(Data_augV5(Uniform-14TFx2-MagFxSh)-LeNet)-100 epochs (dataug:0)- 0 in_it.json",
-        "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx2-MagFxSh)-LeNet)-150epochs(dataug:0)-0in_it-0.json",
+        "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-10in_it-0.json",
+        "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-10in_it-1.json",
+        "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-10in_it-2.json",
         #"res/log/Aug_mod(RandAugUDA(18TFx2-Mag1)-LeNet)-100 epochs (dataug:0)- 0 in_it.json",
     ]
 
@@ -80,7 +82,7 @@ if __name__ == "__main__":
     nb_run=3
     accs = []
     times = []
-    files = ["res/brutus-tests/log/Aug_mod(Data_augV5(Mix1.0-18TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-1in_it-%s.json"%str(run) for run in range(nb_run)]
+    files = ["res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx2-MagFxSh)-LeNet)-150epochs(dataug:0)-0in_it-%s.json"%str(run) for run in range(nb_run)]
     
     for idx, file in enumerate(files):
         #legend+=str(idx)+'-'+file+'\n'
@@ -88,6 +90,7 @@ if __name__ == "__main__":
             data = json.load(json_file)
         accs.append(data['Accuracy'])
         times.append(data['Time'][0])
+        print(idx, data['Accuracy'])
 
     print(files[0], np.mean(accs), np.std(accs), np.mean(times))
     #'''

higher/model.py

@@ -4,9 +4,9 @@ import torch.nn as nn
 import torch.nn.functional as F
 
 ## Basic CNN ##
-class LeNet(nn.Module):
+class LeNet_F(nn.Module):
     def __init__(self, num_inp, num_out):
-        super(LeNet, self).__init__()
+        super(LeNet_F, self).__init__()
         self._params = nn.ParameterDict({
             'w1': nn.Parameter(torch.zeros(20, num_inp, 5, 5)),
             'b1': nn.Parameter(torch.zeros(20)),
@@ -52,6 +52,178 @@ class LeNet(nn.Module):
     def __str__(self):
         return "LeNet"
 
+class LeNet(nn.Module):
+    def __init__(self, num_inp, num_out):
+        super(LeNet, self).__init__()
+        self.conv1 = nn.Conv2d(num_inp, 20, 5)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv2 = nn.Conv2d(20, 50, 5)
+        self.pool2 = nn.MaxPool2d(2, 2)
+        self.fc1 = nn.Linear(5*5*50, 500)
+        self.fc2 = nn.Linear(500, num_out)
+
+    def forward(self, x):
+        x = self.pool(F.relu(self.conv1(x)))
+        x = self.pool2(F.relu(self.conv2(x)))
+        x = x.view(x.size(0), -1)
+        x = F.relu(self.fc1(x))
+        x = self.fc2(x)
+        return x
+
+    def __str__(self):
+        return "LeNet"
+
+## MobileNetv2 ##
+
+def _make_divisible(v, divisor, min_value=None):
+    """
+    This function is taken from the original tf repo.
+    It ensures that all layers have a channel number that is divisible by 8
+    It can be seen here:
+    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py
+    :param v:
+    :param divisor:
+    :param min_value:
+    :return:
+    """
+    if min_value is None:
+        min_value = divisor
+    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)
+    # Make sure that round down does not go down by more than 10%.
+    if new_v < 0.9 * v:
+        new_v += divisor
+    return new_v
+
+
+class ConvBNReLU(nn.Sequential):
+    def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1):
+        padding = (kernel_size - 1) // 2
+        super(ConvBNReLU, self).__init__(
+            nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False),
+            nn.BatchNorm2d(out_planes),
+            nn.ReLU6(inplace=True)
+        )
+
+
+class InvertedResidual(nn.Module):
+    def __init__(self, inp, oup, stride, expand_ratio):
+        super(InvertedResidual, self).__init__()
+        self.stride = stride
+        assert stride in [1, 2]
+
+        hidden_dim = int(round(inp * expand_ratio))
+        self.use_res_connect = self.stride == 1 and inp == oup
+
+        layers = []
+        if expand_ratio != 1:
+            # pw
+            layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1))
+        layers.extend([
+            # dw
+            ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim),
+            # pw-linear
+            nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),
+            nn.BatchNorm2d(oup),
+        ])
+        self.conv = nn.Sequential(*layers)
+
+    def forward(self, x):
+        if self.use_res_connect:
+            return x + self.conv(x)
+        else:
+            return self.conv(x)
+
+
+class MobileNetV2(nn.Module):
+    def __init__(self,
+                 num_classes=1000,
+                 width_mult=1.0,
+                 inverted_residual_setting=None,
+                 round_nearest=8,
+                 block=None):
+        """
+        MobileNet V2 main class
+        Args:
+            num_classes (int): Number of classes
+            width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount
+            inverted_residual_setting: Network structure
+            round_nearest (int): Round the number of channels in each layer to be a multiple of this number
+            Set to 1 to turn off rounding
+            block: Module specifying inverted residual building block for mobilenet
+        """
+        super(MobileNetV2, self).__init__()
+
+        if block is None:
+            block = InvertedResidual
+        input_channel = 32
+        last_channel = 1280
+
+        if inverted_residual_setting is None:
+            inverted_residual_setting = [
+                # t, c, n, s
+                [1, 16, 1, 1],
+                [6, 24, 2, 2],
+                [6, 32, 3, 2],
+                [6, 64, 4, 2],
+                [6, 96, 3, 1],
+                [6, 160, 3, 2],
+                [6, 320, 1, 1],
+            ]
+
+        # only check the first element, assuming user knows t,c,n,s are required
+        if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4:
+            raise ValueError("inverted_residual_setting should be non-empty "
+                             "or a 4-element list, got {}".format(inverted_residual_setting))
+
+        # building first layer
+        input_channel = _make_divisible(input_channel * width_mult, round_nearest)
+        self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest)
+        features = [ConvBNReLU(3, input_channel, stride=2)]
+        # building inverted residual blocks
+        for t, c, n, s in inverted_residual_setting:
+            output_channel = _make_divisible(c * width_mult, round_nearest)
+            for i in range(n):
+                stride = s if i == 0 else 1
+                features.append(block(input_channel, output_channel, stride, expand_ratio=t))
+                input_channel = output_channel
+        # building last several layers
+        features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1))
+        # make it nn.Sequential
+        self.features = nn.Sequential(*features)
+
+        # building classifier
+        self.classifier = nn.Sequential(
+            nn.Dropout(0.2),
+            nn.Linear(self.last_channel, num_classes),
+        )
+
+        # weight initialization
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out')
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.ones_(m.weight)
+                nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.zeros_(m.bias)
+
+    def _forward_impl(self, x):
+        # This exists since TorchScript doesn't support inheritance, so the superclass method
+        # (this one) needs to have a name other than `forward` that can be accessed in a subclass
+        x = self.features(x)
+        x = x.mean([2, 3])
+        x = self.classifier(x)
+        return x
+
+    def forward(self, x):
+        return self._forward_impl(x)
+
+    def __str__(self):
+        return "MobileNetV2"
+
 ## Wide ResNet ##
 #https://github.com/xternalz/WideResNet-pytorch/blob/master/wideresnet.py
 #https://github.com/arcelien/pba/blob/master/pba/wrn.py
@@ -95,7 +267,6 @@ class NetworkBlock(nn.Module):
     def forward(self, x):
         return self.layer(x)
 
-
 #wrn_size: 32 = WRN-28-2 ? 160 = WRN-28-10
 class WideResNet(nn.Module):
     #def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0):

higher/test_dataug.py

@@ -69,21 +69,21 @@ if __name__ == "__main__":
         #'aug_dataset',
         'aug_model'
     }
-    n_inner_iter = 1
+    n_inner_iter = 0
-    epochs = 100
+    epochs = 150
     dataug_epoch_start=0
 
+    model = LeNet(3,10)
+    #model = MobileNetV2(num_classes=10)
+    #model = WideResNet(num_classes=10, wrn_size=32)
 
     #### Classic ####
     if 'classic' in tasks:
         t0 = time.process_time()
-        model = LeNet(3,10).to(device)
+        model = model.to(device)
-        #model = WideResNet(num_classes=10, wrn_size=16).to(device)
-        #model = Augmented_model(Data_augV3(mix_dist=0.0), LeNet(3,10)).to(device)
-        #model.augment(mode=False)
 
-        print(str(model), 'on', device_name)
+        print("{} on {} for {} epochs".format(str(model), device_name, epochs))
-        log= train_classic(model=model, epochs=epochs, print_freq=10)
+        log= train_classic(model=model, epochs=epochs, print_freq=1)
         #log= train_classic_higher(model=model, epochs=epochs)
 
         exec_time=time.process_time() - t0
@@ -116,12 +116,9 @@ if __name__ == "__main__":
         xs, ys = next(iter(dl_train))
         viz_sample_data(imgs=xs, labels=ys, fig_name='samples/data_sample_{}'.format(str(data_train_aug)))
 
-        model = LeNet(3,10).to(device)
+        model = model.to(device)
-        #model = WideResNet(num_classes=10, wrn_size=16).to(device)
-        #model = Augmented_model(Data_augV3(mix_dist=0.0), LeNet(3,10)).to(device)
-        #model.augment(mode=False)
 
-        print(str(model), 'on', device_name)
+        print("{} on {} for {} epochs".format(str(model), device_name, epochs))
         log= train_classic(model=model, epochs=epochs, print_freq=10)
         #log= train_classic_higher(model=model, epochs=epochs)
 
@@ -147,14 +144,13 @@ if __name__ == "__main__":
         t0 = time.process_time()
 
         tf_dict = {k: TF.TF_dict[k] for k in tf_names}
+        #aug_model = Augmented_model(Data_augV6(TF_dict=tf_dict, N_TF=1, mix_dist=0.0, fixed_prob=False, prob_set_size=2, fixed_mag=True, shared_mag=True), model).to(device)
+        aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=2, mix_dist=0.0, fixed_prob=True, fixed_mag=True, shared_mag=True), model).to(device)
+        #aug_model = Augmented_model(RandAug(TF_dict=tf_dict, N_TF=2), model).to(device)
 
-        #aug_model = Augmented_model(Data_augV6(TF_dict=tf_dict, N_TF=1, mix_dist=0.0, fixed_prob=False, prob_set_size=2, fixed_mag=True, shared_mag=True), LeNet(3,10)).to(device)
+        print("{} on {} for {} epochs - {} inner_it".format(str(aug_model), device_name, epochs, n_inner_iter))
-        aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=3, mix_dist=0.0, fixed_prob=False, fixed_mag=False, shared_mag=False), LeNet(3,10)).to(device)
-        #aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=3, mix_dist=0.0, fixed_prob=False, fixed_mag=False, shared_mag=False), WideResNet(num_classes=10, wrn_size=32)).to(device)
-        #aug_model = Augmented_model(RandAug(TF_dict=tf_dict, N_TF=2), WideResNet(num_classes=10, wrn_size=32)).to(device)
-        print(str(aug_model), 'on', device_name)
         #run_simple_dataug(inner_it=n_inner_iter, epochs=epochs)
-        log= run_dist_dataugV2(model=aug_model, epochs=epochs, inner_it=n_inner_iter, dataug_epoch_start=dataug_epoch_start, print_freq=1, KLdiv=True, loss_patience=None)
+        log= run_dist_dataugV2(model=aug_model, epochs=epochs, inner_it=n_inner_iter, dataug_epoch_start=dataug_epoch_start, print_freq=10, KLdiv=True, loss_patience=None)
 
         exec_time=time.process_time() - t0
         ####

higher/train_utils.py

@@ -63,7 +63,8 @@ def train_classic(model, epochs=1, print_freq=1):
             features,labels = features.to(device), labels.to(device)
 
             optim.zero_grad()
-            pred = model.forward(features)
+            logits = model.forward(features)
+            pred = F.log_softmax(logits, dim=1)
             loss = F.cross_entropy(pred,labels)
             loss.backward()
             optim.step()
@@ -125,7 +126,8 @@ def train_classic_higher(model, epochs=1):
             features,labels = features.to(device), labels.to(device)
 
             #optim.zero_grad()
-            pred = fmodel.forward(features)
+            logits = model.forward(features)
+            pred = F.log_softmax(logits, dim=1)
             loss = F.cross_entropy(pred,labels)
             #.backward()
             #optim.step()
@@ -550,7 +552,7 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f
     dl_val_it = iter(dl_val)
 
     #if inner_it!=0: 
-    meta_opt = torch.optim.Adam(model['data_aug'].parameters(), lr=1e-2)
+    meta_opt = torch.optim.Adam(model['data_aug'].parameters(), lr=1e-2) #lr=1e-2
     inner_opt = torch.optim.SGD(model['model'].parameters(), lr=1e-2, momentum=0.9)
 
     high_grad_track = True
@@ -589,11 +591,10 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f
             #    final_loss += loss*fmodel['data_aug']['prob'][tf_idx] #Take it in the forward function ? 
             #loss = final_loss 
             
-            #KLdiv=False
             if(not KLdiv):
             #Methode uniforme
                 logits = fmodel(xs)  # modified `params` can also be passed as a kwarg
-                loss = F.cross_entropy(logits, ys, reduction='none')  # no need to call loss.backwards()
+                loss = F.cross_entropy(F.log_softmax(logits, dim=1), ys, reduction='none')  # no need to call loss.backwards()
 
                 if fmodel._data_augmentation: #Weight loss
                     w_loss = fmodel['data_aug'].loss_weight()#.to(device)
@@ -612,18 +613,15 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f
                     aug_logits = fmodel(xs)
                     log_aug=F.log_softmax(aug_logits, dim=1)
                     #KL div w/ logits
-                    aug_loss = sup_logits*(log_sup-log_aug)
+                    aug_loss = F.softmax(sup_logits, dim=1)*(log_sup-log_aug)
                     aug_loss=aug_loss.sum(dim=-1)
                     #aug_loss = F.kl_div(aug_logits, sup_logits, reduction='none') #Similarite predictions (distributions)
 
-                    w_loss = fmodel['data_aug'].loss_weight()#.unsqueeze(dim=1).expand(-1,10) #Weight loss
+                    w_loss = fmodel['data_aug'].loss_weight() #Weight loss
                     aug_loss = (w_loss * aug_loss).mean()
                     unsupp_coeff = 1
                     loss += aug_loss * unsupp_coeff
 
-                    print('TF Proba :', model['data_aug']['prob'].data)
-
-
             #to visualize computational graph
             #print_graph(loss)
 
@@ -637,7 +635,6 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f
                 #print("meta")
                 #Peu utile si high_grad_track = False
                 val_loss = compute_vaLoss(model=fmodel, dl_it=dl_val_it, dl=dl_val) + fmodel['data_aug'].reg_loss()
-
                 #print_graph(val_loss)
 
                 val_loss.backward()
@@ -685,7 +682,7 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f
             print('TF Mag :', model['data_aug']['mag'].data)
             #print('Mag grad',model['data_aug']['mag'].grad)
             #print('Reg loss:', model['data_aug'].reg_loss().item())
-            print('Aug loss', aug_loss.item())
+            #print('Aug loss', aug_loss.item())
         #############
         #### Log ####
         #print(type(model['data_aug']) is dataug.Data_augV5)

higher/utils.py

@@ -9,6 +9,16 @@ from torchviz import make_dot
 import torch
 import torch.nn.functional as F
 
+import time
+
+class timer():
+    def __init__(self):
+        self._start_time=time.time()
+    def exec_time(self):
+        end = time.time()
+        res = end-self._start_time
+        self._start_time=end
+        return res
 
 def print_graph(PyTorch_obj, fig_name='graph'):
     graph=make_dot(PyTorch_obj) #Loss give the whole graph