diff --git a/higher/smart_aug/model.py b/higher/smart_aug/LeNet.py similarity index 67% rename from higher/smart_aug/model.py rename to higher/smart_aug/LeNet.py index 4b377ce..cb35492 100755 --- a/higher/smart_aug/model.py +++ b/higher/smart_aug/LeNet.py @@ -36,3 +36,20 @@ class LeNet(nn.Module): """ return "LeNet" + +#MNIST +class MLPNet(nn.Module): + def __init__(self): + super(MLPNet, self).__init__() + self.fc1 = nn.Linear(28*28, 500) + self.fc2 = nn.Linear(500, 256) + self.fc3 = nn.Linear(256, 10) + def forward(self, x): + x = x.view(-1, 28*28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + def name(self): + return "MLP" \ No newline at end of file diff --git a/higher/smart_aug/benchmark.py b/higher/smart_aug/benchmark.py index d71baa4..bc19929 100644 --- a/higher/smart_aug/benchmark.py +++ b/higher/smart_aug/benchmark.py @@ -1,4 +1,3 @@ -from model import * from dataug import * #from utils import * from train_utils import * @@ -19,9 +18,9 @@ optim_param={ } } -res_folder="../res/benchmark/CIFAR10" -epochs= 200 -dataug_epoch_starts=0 +res_folder="../res/benchmark/CIFAR10/" +epochs= 150 +dataug_epoch_start=0 # Use available TF (see transformations.py) tf_names = [ @@ -131,7 +130,6 @@ if __name__ == "__main__": ### HP Search ### inner_its = [1] dist_mix = [0.0, 0.5, 0.8, 1.0] - dataug_epoch_starts= [0] N_seq_TF= [2, 3, 4] mag_setup = [(True,True), (False, False)] #(FxSh, Independant) #prob_setup = [True, False] @@ -154,7 +152,7 @@ if __name__ == "__main__": t0 = time.process_time() - model = getattr(model_list.keys()[0], 'resnet18')(pretrained=False) + model = getattr(models.resnet, 'resnet18')(pretrained=False) model = Higher_model(model) #run_dist_dataugV3 aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=n_tf, mix_dist=dist, fixed_prob=p_setup, fixed_mag=m_setup[0], shared_mag=m_setup[1]), model).to(device) #aug_model = Augmented_model(RandAug(TF_dict=tf_dict, N_TF=2), model).to(device) @@ -177,7 +175,7 @@ if __name__ == "__main__": out = {"Accuracy": max([x["acc"] for x in log]), "Time": (np.mean(times),np.std(times), exec_time), 'Optimizer': optim_param, "Device": device_name, "Param_names": aug_model.TF_names(), "Log": log} print(str(aug_model),": acc", out["Accuracy"], "in:", out["Time"][0], "+/-", out["Time"][1]) filename = "{}-{} epochs (dataug:{})- {} in_it-{}".format(str(aug_model),epochs,dataug_epoch_start,n_inner_iter, run) - with open("../res/log/%s.json" % filename, "w+") as f: + with open(res_folder+"log/%s.json" % filename, "w+") as f: try: json.dump(out, f, indent=True) print('Log :\"',f.name, '\" saved !') diff --git a/higher/smart_aug/datasets.py b/higher/smart_aug/datasets.py index 9d4b5a6..26a0975 100755 --- a/higher/smart_aug/datasets.py +++ b/higher/smart_aug/datasets.py @@ -27,25 +27,26 @@ pin_memory=False #True :+ GPU memory / + Lent #]) transform = torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), - #torchvision.transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), #CIFAR10 +# torchvision.transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), #CIFAR10 ]) -#data_train = torchvision.datasets.MNIST( -# "./data", train=True, download=True, -# transform=torchvision.transforms.Compose([ -# #torchvision.transforms.RandomAffine(degrees=180, translate=None, scale=None, shear=None, resample=False, fillcolor=0), -# torchvision.transforms.ToTensor() -# ]) -#) -#data_test = torchvision.datasets.MNIST( -# "./data", train=False, download=True, transform=torchvision.transforms.ToTensor() -#) +from RandAugment import RandAugment +# Add RandAugment with N, M(hyperparameter) +transform_train = torchvision.transforms.Compose([ + #transforms.RandomHorizontalFlip(), + #transforms.RandomVerticalFlip(), + torchvision.transforms.ToTensor(), +]) +#transform_train.transforms.insert(0, RandAugment(n=2, m=30)) ### Classic Dataset ### -#Training data -data_train = torchvision.datasets.CIFAR10("../data", train=True, download=download_data, transform=transform) -#data_val = torchvision.datasets.CIFAR10("../data", train=True, download=download_data, transform=transform) -#Testing data +#MNIST +#data_train = torchvision.datasets.MNIST("../data", train=True, download=True, transform=transform_train) +#data_val = torchvision.datasets.MNIST("../data", train=True, download=True, transform=transform) +#data_test = torchvision.datasets.MNIST("../data", train=False, download=True, transform=transform) +#CIFAR +data_train = torchvision.datasets.CIFAR10("../data", train=True, download=download_data, transform=transform_train) +data_val = torchvision.datasets.CIFAR10("../data", train=True, download=download_data, transform=transform) data_test = torchvision.datasets.CIFAR10("../data", train=False, download=download_data, transform=transform) train_subset_indices=range(int(len(data_train)/2)) @@ -54,5 +55,5 @@ val_subset_indices=range(int(len(data_train)/2),len(data_train)) #val_subset_indices=range(BATCH_SIZE*10, BATCH_SIZE*20) dl_train = torch.utils.data.DataLoader(data_train, batch_size=BATCH_SIZE, shuffle=False, sampler=SubsetRandomSampler(train_subset_indices), num_workers=num_workers, pin_memory=pin_memory) -dl_val = torch.utils.data.DataLoader(data_train, batch_size=BATCH_SIZE, shuffle=False, sampler=SubsetRandomSampler(val_subset_indices), num_workers=num_workers, pin_memory=pin_memory) +dl_val = torch.utils.data.DataLoader(data_val, batch_size=BATCH_SIZE, shuffle=False, sampler=SubsetRandomSampler(val_subset_indices), num_workers=num_workers, pin_memory=pin_memory) dl_test = torch.utils.data.DataLoader(data_test, batch_size=TEST_SIZE, shuffle=False, num_workers=num_workers, pin_memory=pin_memory) diff --git a/higher/smart_aug/dataug.py b/higher/smart_aug/dataug.py index 246a336..c11610e 100755 --- a/higher/smart_aug/dataug.py +++ b/higher/smart_aug/dataug.py @@ -35,7 +35,7 @@ class Data_augV5(nn.Module): #Optimisation jointe (mag, proba) _TF (list) : List of TF names. _nb_tf (int) : Number of TF used. _N_seqTF (int) : Number of TF to be applied sequentially to each inputs - _shared_mag (bool) : Wether to share a single magnitude parameters for all TF. + _shared_mag (bool) : Wether to share a single magnitude parameters for all TF. Beware using shared mag with basic color TF as their lowest magnitude is at PARAMETER_MAX/2. _fixed_mag (bool): Wether to lock the TF magnitudes. _fixed_prob (bool): Wether to lock the TF probabilies. _samples (list): Sampled TF index during last forward pass. @@ -320,7 +320,7 @@ class Data_augV7(nn.Module): #Proba sequentielles _TF (list) : List of TF names. _nb_tf (int) : Number of TF used. _N_seqTF (int) : Number of TF to be applied sequentially to each inputs - _shared_mag (bool) : Wether to share a single magnitude parameters for all TF. + _shared_mag (bool) : Wether to share a single magnitude parameters for all TF. Beware using shared mag with basic color TF as their lowest magnitude is at PARAMETER_MAX/2. _fixed_mag (bool): Wether to lock the TF magnitudes. _fixed_prob (bool): Wether to lock the TF probabilies. _samples (list): Sampled TF index during last forward pass. diff --git a/higher/smart_aug/test_dataug.py b/higher/smart_aug/test_dataug.py index b7b339a..ade803f 100755 --- a/higher/smart_aug/test_dataug.py +++ b/higher/smart_aug/test_dataug.py @@ -2,7 +2,7 @@ """ -from model import * +from LeNet import * from dataug import * #from utils import * from train_utils import * @@ -13,19 +13,19 @@ tf_names = [ 'Identity', 'FlipUD', 'FlipLR', - 'Rotate', - 'TranslateX', - 'TranslateY', - 'ShearX', - 'ShearY', + #'Rotate', + #'TranslateX', + #'TranslateY', + #'ShearX', + #'ShearY', ## Color TF (Expect image in the range of [0, 1]) ## - 'Contrast', - 'Color', - 'Brightness', - 'Sharpness', - 'Posterize', - 'Solarize', #=>Image entre [0,1] #Pas opti pour des batch + #'Contrast', + #'Color', + #'Brightness', + #'Sharpness', + #'Posterize', + #'Solarize', #=>Image entre [0,1] #Pas opti pour des batch #Color TF (Common mag scale) #'+Contrast', @@ -76,7 +76,6 @@ if __name__ == "__main__": tasks={ #'classic', 'aug_model' - #'aug_dataset', #Moved to old code } #Parameters n_inner_iter = 1 @@ -131,7 +130,7 @@ if __name__ == "__main__": tf_dict = {k: TF.TF_dict[k] for k in tf_names} model = Higher_model(model) #run_dist_dataugV3 - aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=2, mix_dist=0.8, fixed_prob=False, fixed_mag=False, shared_mag=False), model).to(device) + aug_model = Augmented_model(Data_augV5(TF_dict=tf_dict, N_TF=3, mix_dist=0.8, fixed_prob=False, fixed_mag=False, shared_mag=False), model).to(device) #aug_model = Augmented_model(RandAug(TF_dict=tf_dict, N_TF=2), model).to(device) print("{} on {} for {} epochs - {} inner_it".format(str(aug_model), device_name, epochs, n_inner_iter)) @@ -140,7 +139,7 @@ if __name__ == "__main__": inner_it=n_inner_iter, dataug_epoch_start=dataug_epoch_start, opt_param=optim_param, - print_freq=1, + print_freq=20, unsup_loss=1, hp_opt=False, save_sample_freq=None) @@ -164,56 +163,4 @@ if __name__ == "__main__": print("Failed to plot res") print('Execution Time : %.00f '%(exec_time)) - print('-'*9) - - #### Augmented Dataset #### - ''' - if 'aug_dataset' in tasks: - - t0 = time.process_time() - - #data_train_aug = AugmentedDataset("./data", train=True, download=download_data, transform=transform, subset=(0,int(len(data_train)/2))) - #data_train_aug.augement_data(aug_copy=30) - #print(data_train_aug) - #dl_train = torch.utils.data.DataLoader(data_train_aug, batch_size=BATCH_SIZE, shuffle=True) - - #xs, ys = next(iter(dl_train)) - #viz_sample_data(imgs=xs, labels=ys, fig_name='samples/data_sample_{}'.format(str(data_train_aug))) - - #model = model.to(device) - - #print("{} on {} for {} epochs".format(str(model), device_name, epochs)) - #log= train_classic(model=model, epochs=epochs, print_freq=10) - ##log= train_classic_higher(model=model, epochs=epochs) - - data_train_aug = AugmentedDatasetV2("./data", train=True, download=download_data, transform=transform, subset=(0,int(len(data_train)/2))) - data_train_aug.augement_data(aug_copy=1) - print(data_train_aug) - unsup_ratio = 5 - dl_unsup = torch.utils.data.DataLoader(data_train_aug, batch_size=BATCH_SIZE*unsup_ratio, shuffle=True, num_workers=num_workers, pin_memory=pin_memory) - - unsup_xs, sup_xs, ys = next(iter(dl_unsup)) - viz_sample_data(imgs=sup_xs, labels=ys, fig_name='samples/data_sample_{}'.format(str(data_train_aug))) - viz_sample_data(imgs=unsup_xs, labels=ys, fig_name='samples/data_sample_{}_unsup'.format(str(data_train_aug))) - - model = model.to(device) - - print("{} on {} for {} epochs".format(str(model), device_name, epochs)) - log= train_UDA(model=model, dl_unsup=dl_unsup, epochs=epochs, opt_param=optim_param, print_freq=10) - - exec_time=time.process_time() - t0 - #### - print('-'*9) - times = [x["time"] for x in log] - out = {"Accuracy": max([x["acc"] for x in log]), "Time": (np.mean(times),np.std(times), exec_time), 'Optimizer': optim_param['Inner'], "Device": device_name, "Param_names": data_train_aug._TF, "Log": log} - print(str(model),": acc", out["Accuracy"], "in:", out["Time"][0], "+/-", out["Time"][1]) - filename = "{}-{}-{} epochs".format(str(data_train_aug),str(model),epochs) - with open("res/log/%s.json" % filename, "w+") as f: - json.dump(out, f, indent=True) - print('Log :\"',f.name, '\" saved !') - - plot_res(log, fig_name="res/"+filename) - - print('Execution Time : %.00f '%(exec_time)) - print('-'*9) - ''' \ No newline at end of file + print('-'*9) \ No newline at end of file diff --git a/higher/smart_aug/train_utils.py b/higher/smart_aug/train_utils.py index 31d42e3..02f8c49 100755 --- a/higher/smart_aug/train_utils.py +++ b/higher/smart_aug/train_utils.py @@ -144,6 +144,7 @@ def train_classic(model, opt_param, epochs=1, print_freq=1): #print_torch_mem("Start epoch") t0 = time.process_time() for i, (features, labels) in enumerate(dl_train): + #viz_sample_data(imgs=features, labels=labels, fig_name='../samples/data_sample_epoch{}_noTF'.format(epoch)) #print_torch_mem("Start iter") features,labels = features.to(device), labels.to(device) @@ -277,7 +278,7 @@ def run_dist_dataugV3(model, opt_param, epochs=1, inner_it=1, dataug_epoch_start meta_opt.step() #Adjust Hyper-parameters - model['data_aug'].adjust_param(soft=False) #Contrainte sum(proba)=1 + model['data_aug'].adjust_param() #Contrainte sum(proba)=1 if hp_opt: for param_group in diffopt.param_groups: for param in list(opt_param['Inner'].keys())[1:]: @@ -289,7 +290,7 @@ def run_dist_dataugV3(model, opt_param, epochs=1, inner_it=1, dataug_epoch_start meta_opt.zero_grad() elif not high_grad_track: - diffopt.detach_() + #diffopt.detach_() model['model'].detach_() tf = time.process_time()