From d68034eec1e4370e7d23b9827bfe3a295695572f Mon Sep 17 00:00:00 2001 From: "Harle, Antoine (Contracteur)" Date: Fri, 6 Dec 2019 14:13:28 -0500 Subject: [PATCH] Tests consomation memoire/temps + methode KL divergence (UDA) --- higher/compare_res.py | 7 +- higher/model.py | 177 +++++++++++++++++++++++++++++++++++++++++- higher/test_dataug.py | 34 ++++---- higher/train_utils.py | 23 +++--- higher/utils.py | 10 +++ 5 files changed, 214 insertions(+), 37 deletions(-) diff --git a/higher/compare_res.py b/higher/compare_res.py index 65d87ee..fbe4949 100755 --- a/higher/compare_res.py +++ b/higher/compare_res.py @@ -6,7 +6,9 @@ if __name__ == "__main__": files=[ #"res/good_TF_tests/log/Aug_mod(Data_augV5(Mix0.5-14TFx2-MagFxSh)-LeNet)-100 epochs (dataug:0)- 0 in_it.json", #"res/good_TF_tests/log/Aug_mod(Data_augV5(Uniform-14TFx2-MagFxSh)-LeNet)-100 epochs (dataug:0)- 0 in_it.json", - "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx2-MagFxSh)-LeNet)-150epochs(dataug:0)-0in_it-0.json", + "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-10in_it-0.json", + "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-10in_it-1.json", + "res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-10in_it-2.json", #"res/log/Aug_mod(RandAugUDA(18TFx2-Mag1)-LeNet)-100 epochs (dataug:0)- 0 in_it.json", ] @@ -80,7 +82,7 @@ if __name__ == "__main__": nb_run=3 accs = [] times = [] - files = ["res/brutus-tests/log/Aug_mod(Data_augV5(Mix1.0-18TFx3-MagFxSh)-LeNet)-150epochs(dataug:0)-1in_it-%s.json"%str(run) for run in range(nb_run)] + files = ["res/brutus-tests/log/Aug_mod(Data_augV5(Uniform-14TFx2-MagFxSh)-LeNet)-150epochs(dataug:0)-0in_it-%s.json"%str(run) for run in range(nb_run)] for idx, file in enumerate(files): #legend+=str(idx)+'-'+file+'\n' @@ -88,6 +90,7 @@ if __name__ == "__main__": data = json.load(json_file) accs.append(data['Accuracy']) times.append(data['Time'][0]) + print(idx, data['Accuracy']) print(files[0], np.mean(accs), np.std(accs), np.mean(times)) #''' \ No newline at end of file diff --git a/higher/model.py b/higher/model.py index 3b466fe..979a85d 100755 --- a/higher/model.py +++ b/higher/model.py @@ -4,9 +4,9 @@ import torch.nn as nn import torch.nn.functional as F ## Basic CNN ## -class LeNet(nn.Module): +class LeNet_F(nn.Module): def __init__(self, num_inp, num_out): - super(LeNet, self).__init__() + super(LeNet_F, self).__init__() self._params = nn.ParameterDict({ 'w1': nn.Parameter(torch.zeros(20, num_inp, 5, 5)), 'b1': nn.Parameter(torch.zeros(20)), @@ -52,6 +52,178 @@ class LeNet(nn.Module): def __str__(self): return "LeNet" +class LeNet(nn.Module): + def __init__(self, num_inp, num_out): + super(LeNet, self).__init__() + self.conv1 = nn.Conv2d(num_inp, 20, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(20, 50, 5) + self.pool2 = nn.MaxPool2d(2, 2) + self.fc1 = nn.Linear(5*5*50, 500) + self.fc2 = nn.Linear(500, num_out) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool2(F.relu(self.conv2(x))) + x = x.view(x.size(0), -1) + x = F.relu(self.fc1(x)) + x = self.fc2(x) + return x + + def __str__(self): + return "LeNet" + +## MobileNetv2 ## + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + padding = (kernel_size - 1) // 2 + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_planes), + nn.ReLU6(inplace=True) + ) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, + num_classes=1000, + width_mult=1.0, + inverted_residual_setting=None, + round_nearest=8, + block=None): + """ + MobileNet V2 main class + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + """ + super(MobileNetV2, self).__init__() + + if block is None: + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + + if inverted_residual_setting is None: + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(3, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(0.2), + nn.Linear(self.last_channel, num_classes), + ) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + x = self.features(x) + x = x.mean([2, 3]) + x = self.classifier(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + def __str__(self): + return "MobileNetV2" + ## Wide ResNet ## #https://github.com/xternalz/WideResNet-pytorch/blob/master/wideresnet.py #https://github.com/arcelien/pba/blob/master/pba/wrn.py @@ -95,7 +267,6 @@ class NetworkBlock(nn.Module): def forward(self, x): return self.layer(x) - #wrn_size: 32 = WRN-28-2 ? 160 = WRN-28-10 class WideResNet(nn.Module): #def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0): diff --git a/higher/test_dataug.py b/higher/test_dataug.py index 6679c39..ec3dbce 100755 --- a/higher/test_dataug.py +++ b/higher/test_dataug.py @@ -69,21 +69,21 @@ if __name__ == "__main__": #'aug_dataset', 'aug_model' } - n_inner_iter = 1 - epochs = 100 + n_inner_iter = 0 + epochs = 150 dataug_epoch_start=0 + model = LeNet(3,10) + #model = MobileNetV2(num_classes=10) + #model = WideResNet(num_classes=10, wrn_size=32) #### Classic #### if 'classic' in tasks: t0 = time.process_time() - model = LeNet(3,10).to(device) - #model = WideResNet(num_classes=10, wrn_size=16).to(device) - #model = Augmented_model(Data_augV3(mix_dist=0.0), LeNet(3,10)).to(device) - #model.augment(mode=False) + model = model.to(device) - print(str(model), 'on', device_name) - log= train_classic(model=model, epochs=epochs, print_freq=10) + print("{} on {} for {} epochs".format(str(model), device_name, epochs)) + log= train_classic(model=model, epochs=epochs, print_freq=1) #log= train_classic_higher(model=model, epochs=epochs) exec_time=time.process_time() - t0 @@ -116,12 +116,9 @@ if __name__ == "__main__": xs, ys = next(iter(dl_train)) viz_sample_data(imgs=xs, labels=ys, fig_name='samples/data_sample_{}'.format(str(data_train_aug))) - model = LeNet(3,10).to(device) - #model = WideResNet(num_classes=10, wrn_size=16).to(device) - #model = Augmented_model(Data_augV3(mix_dist=0.0), LeNet(3,10)).to(device) - #model.augment(mode=False) + model = model.to(device) - print(str(model), 'on', device_name) + print("{} on {} for {} epochs".format(str(model), device_name, epochs)) log= train_classic(model=model, epochs=epochs, print_freq=10) #log= train_classic_higher(model=model, epochs=epochs) @@ -147,14 +144,13 @@ if __name__ == "__main__": t0 = time.process_time() tf_dict = {k: TF.TF_dict[k] for k in tf_names} + #aug_model = Augmented_model(Data_augV6(TF_dict=tf_dict, N_TF=1, mix_dist=0.0, fixed_prob=False, prob_set_size=2, fixed_mag=True, shared_mag=True), model).to(device) + aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=2, mix_dist=0.0, fixed_prob=True, fixed_mag=True, shared_mag=True), model).to(device) + #aug_model = Augmented_model(RandAug(TF_dict=tf_dict, N_TF=2), model).to(device) - #aug_model = Augmented_model(Data_augV6(TF_dict=tf_dict, N_TF=1, mix_dist=0.0, fixed_prob=False, prob_set_size=2, fixed_mag=True, shared_mag=True), LeNet(3,10)).to(device) - aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=3, mix_dist=0.0, fixed_prob=False, fixed_mag=False, shared_mag=False), LeNet(3,10)).to(device) - #aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=3, mix_dist=0.0, fixed_prob=False, fixed_mag=False, shared_mag=False), WideResNet(num_classes=10, wrn_size=32)).to(device) - #aug_model = Augmented_model(RandAug(TF_dict=tf_dict, N_TF=2), WideResNet(num_classes=10, wrn_size=32)).to(device) - print(str(aug_model), 'on', device_name) + print("{} on {} for {} epochs - {} inner_it".format(str(aug_model), device_name, epochs, n_inner_iter)) #run_simple_dataug(inner_it=n_inner_iter, epochs=epochs) - log= run_dist_dataugV2(model=aug_model, epochs=epochs, inner_it=n_inner_iter, dataug_epoch_start=dataug_epoch_start, print_freq=1, KLdiv=True, loss_patience=None) + log= run_dist_dataugV2(model=aug_model, epochs=epochs, inner_it=n_inner_iter, dataug_epoch_start=dataug_epoch_start, print_freq=10, KLdiv=True, loss_patience=None) exec_time=time.process_time() - t0 #### diff --git a/higher/train_utils.py b/higher/train_utils.py index 75335bb..fb14fdb 100755 --- a/higher/train_utils.py +++ b/higher/train_utils.py @@ -63,7 +63,8 @@ def train_classic(model, epochs=1, print_freq=1): features,labels = features.to(device), labels.to(device) optim.zero_grad() - pred = model.forward(features) + logits = model.forward(features) + pred = F.log_softmax(logits, dim=1) loss = F.cross_entropy(pred,labels) loss.backward() optim.step() @@ -125,7 +126,8 @@ def train_classic_higher(model, epochs=1): features,labels = features.to(device), labels.to(device) #optim.zero_grad() - pred = fmodel.forward(features) + logits = model.forward(features) + pred = F.log_softmax(logits, dim=1) loss = F.cross_entropy(pred,labels) #.backward() #optim.step() @@ -550,7 +552,7 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f dl_val_it = iter(dl_val) #if inner_it!=0: - meta_opt = torch.optim.Adam(model['data_aug'].parameters(), lr=1e-2) + meta_opt = torch.optim.Adam(model['data_aug'].parameters(), lr=1e-2) #lr=1e-2 inner_opt = torch.optim.SGD(model['model'].parameters(), lr=1e-2, momentum=0.9) high_grad_track = True @@ -589,11 +591,10 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f # final_loss += loss*fmodel['data_aug']['prob'][tf_idx] #Take it in the forward function ? #loss = final_loss - #KLdiv=False if(not KLdiv): #Methode uniforme logits = fmodel(xs) # modified `params` can also be passed as a kwarg - loss = F.cross_entropy(logits, ys, reduction='none') # no need to call loss.backwards() + loss = F.cross_entropy(F.log_softmax(logits, dim=1), ys, reduction='none') # no need to call loss.backwards() if fmodel._data_augmentation: #Weight loss w_loss = fmodel['data_aug'].loss_weight()#.to(device) @@ -612,18 +613,15 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f aug_logits = fmodel(xs) log_aug=F.log_softmax(aug_logits, dim=1) #KL div w/ logits - aug_loss = sup_logits*(log_sup-log_aug) + aug_loss = F.softmax(sup_logits, dim=1)*(log_sup-log_aug) aug_loss=aug_loss.sum(dim=-1) #aug_loss = F.kl_div(aug_logits, sup_logits, reduction='none') #Similarite predictions (distributions) - w_loss = fmodel['data_aug'].loss_weight()#.unsqueeze(dim=1).expand(-1,10) #Weight loss + w_loss = fmodel['data_aug'].loss_weight() #Weight loss aug_loss = (w_loss * aug_loss).mean() unsupp_coeff = 1 loss += aug_loss * unsupp_coeff - print('TF Proba :', model['data_aug']['prob'].data) - - #to visualize computational graph #print_graph(loss) @@ -637,11 +635,10 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f #print("meta") #Peu utile si high_grad_track = False val_loss = compute_vaLoss(model=fmodel, dl_it=dl_val_it, dl=dl_val) + fmodel['data_aug'].reg_loss() - #print_graph(val_loss) val_loss.backward() - + countcopy+=1 model_copy(src=fmodel, dst=model) optim_copy(dopt=diffopt, opt=inner_opt) @@ -685,7 +682,7 @@ def run_dist_dataugV2(model, epochs=1, inner_it=0, dataug_epoch_start=0, print_f print('TF Mag :', model['data_aug']['mag'].data) #print('Mag grad',model['data_aug']['mag'].grad) #print('Reg loss:', model['data_aug'].reg_loss().item()) - print('Aug loss', aug_loss.item()) + #print('Aug loss', aug_loss.item()) ############# #### Log #### #print(type(model['data_aug']) is dataug.Data_augV5) diff --git a/higher/utils.py b/higher/utils.py index 1f2d0e6..ea81ea3 100755 --- a/higher/utils.py +++ b/higher/utils.py @@ -9,6 +9,16 @@ from torchviz import make_dot import torch import torch.nn.functional as F +import time + +class timer(): + def __init__(self): + self._start_time=time.time() + def exec_time(self): + end = time.time() + res = end-self._start_time + self._start_time=end + return res def print_graph(PyTorch_obj, fig_name='graph'): graph=make_dot(PyTorch_obj) #Loss give the whole graph