Initial Commit

FAR-HO/augmentation_transforms.py

@@ -0,0 +1,456 @@
+# Copyright 2018 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Transforms used in the Augmentation Policies."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import inspect
+import random
+import numpy as np
+# pylint:disable=g-multiple-import
+from PIL import ImageOps, ImageEnhance, ImageFilter, Image
+# pylint:enable=g-multiple-import
+
+
+IMAGE_SIZE = 28
+# What is the dataset mean and std of the images on the training set
+MEANS = [0.49139968, 0.48215841, 0.44653091]
+STDS = [0.24703223, 0.24348513, 0.26158784]
+PARAMETER_MAX = 10  # What is the max 'level' a transform could be predicted
+
+
+def random_flip(x):
+  """Flip the input x horizontally with 50% probability."""
+  if np.random.rand(1)[0] > 0.5:
+    return np.fliplr(x)
+  return x
+
+
+def zero_pad_and_crop(img, amount=4):
+  """Zero pad by `amount` zero pixels on each side then take a random crop.
+
+  Args:
+    img: numpy image that will be zero padded and cropped.
+    amount: amount of zeros to pad `img` with horizontally and verically.
+
+  Returns:
+    The cropped zero padded img. The returned numpy array will be of the same
+    shape as `img`.
+  """
+  padded_img = np.zeros((img.shape[0] + amount * 2, img.shape[1] + amount * 2,
+                         img.shape[2]))
+  padded_img[amount:img.shape[0] + amount, amount:
+             img.shape[1] + amount, :] = img
+  top = np.random.randint(low=0, high=2 * amount)
+  left = np.random.randint(low=0, high=2 * amount)
+  new_img = padded_img[top:top + img.shape[0], left:left + img.shape[1], :]
+  return new_img
+
+
+def create_cutout_mask(img_height, img_width, num_channels, size):
+  """Creates a zero mask used for cutout of shape `img_height` x `img_width`.
+
+  Args:
+    img_height: Height of image cutout mask will be applied to.
+    img_width: Width of image cutout mask will be applied to.
+    num_channels: Number of channels in the image.
+    size: Size of the zeros mask.
+
+  Returns:
+    A mask of shape `img_height` x `img_width` with all ones except for a
+    square of zeros of shape `size` x `size`. This mask is meant to be
+    elementwise multiplied with the original image. Additionally returns
+    the `upper_coord` and `lower_coord` which specify where the cutout mask
+    will be applied.
+  """
+  assert img_height == img_width
+
+  # Sample center where cutout mask will be applied
+  height_loc = np.random.randint(low=0, high=img_height)
+  width_loc = np.random.randint(low=0, high=img_width)
+
+  # Determine upper right and lower left corners of patch
+  upper_coord = (max(0, height_loc - size // 2), max(0, width_loc - size // 2))
+  lower_coord = (min(img_height, height_loc + size // 2),
+                 min(img_width, width_loc + size // 2))
+  mask_height = lower_coord[0] - upper_coord[0]
+  mask_width = lower_coord[1] - upper_coord[1]
+  assert mask_height > 0
+  assert mask_width > 0
+
+  mask = np.ones((img_height, img_width, num_channels))
+  zeros = np.zeros((mask_height, mask_width, num_channels))
+  mask[upper_coord[0]:lower_coord[0], upper_coord[1]:lower_coord[1], :] = (
+      zeros)
+  return mask, upper_coord, lower_coord
+
+
+def cutout_numpy(img, size=16):
+  """Apply cutout with mask of shape `size` x `size` to `img`.
+
+  The cutout operation is from the paper https://arxiv.org/abs/1708.04552.
+  This operation applies a `size`x`size` mask of zeros to a random location
+  within `img`.
+
+  Args:
+    img: Numpy image that cutout will be applied to.
+    size: Height/width of the cutout mask that will be
+
+  Returns:
+    A numpy tensor that is the result of applying the cutout mask to `img`.
+  """
+  img_height, img_width, num_channels = (img.shape[0], img.shape[1],
+                                         img.shape[2])
+  assert len(img.shape) == 3
+  mask, _, _ = create_cutout_mask(img_height, img_width, num_channels, size)
+  return img * mask
+
+
+def float_parameter(level, maxval):
+  """Helper function to scale `val` between 0 and maxval .
+
+  Args:
+    level: Level of the operation that will be between [0, `PARAMETER_MAX`].
+    maxval: Maximum value that the operation can have. This will be scaled
+      to level/PARAMETER_MAX.
+
+  Returns:
+    A float that results from scaling `maxval` according to `level`.
+  """
+  return float(level) * maxval / PARAMETER_MAX
+
+
+def int_parameter(level, maxval):
+  """Helper function to scale `val` between 0 and maxval .
+
+  Args:
+    level: Level of the operation that will be between [0, `PARAMETER_MAX`].
+    maxval: Maximum value that the operation can have. This will be scaled
+      to level/PARAMETER_MAX.
+
+  Returns:
+    An int that results from scaling `maxval` according to `level`.
+  """
+  return int(level * maxval / PARAMETER_MAX)
+
+
+def pil_wrap(img):
+  """Convert the `img` numpy tensor to a PIL Image."""
+  return Image.fromarray(
+      np.uint8((img * STDS + MEANS) * 255.0)).convert('RGBA')
+
+
+def pil_unwrap(pil_img):
+  """Converts the PIL img to a numpy array."""
+  pic_array = (np.array(pil_img.getdata()).reshape((IMAGE_SIZE, IMAGE_SIZE, 4)) / 255.0)
+  i1, i2 = np.where(pic_array[:, :, 3] == 0)
+  pic_array = (pic_array[:, :, :3] - MEANS) / STDS
+  pic_array[i1, i2] = [0, 0, 0]
+  return pic_array
+
+
+def apply_policy(policy, img):
+  """Apply the `policy` to the numpy `img`.
+
+  Args:
+    policy: A list of tuples with the form (name, probability, level) where
+      `name` is the name of the augmentation operation to apply, `probability`
+      is the probability of applying the operation and `level` is what strength
+      the operation to apply.
+    img: Numpy image that will have `policy` applied to it.
+
+  Returns:
+    The result of applying `policy` to `img`.
+  """
+  #print('img shape :',img.shape)
+  #print('Policy len :',len(policy))
+  pil_img = pil_wrap(img)
+  for xform in policy:
+    #print('xform :', len(xform))
+    assert len(xform) == 3
+    name, probability, level = xform
+    #xform_fn = NAME_TO_TRANSFORM[name].pil_transformer(probability, level)
+    xform_fn = NAME_TO_TRANSFORM[name].pil_transformer(probability.eval(), level)
+    pil_img = xform_fn(pil_img)
+  return pil_unwrap(pil_img)
+
+
+class TransformFunction(object):
+  """Wraps the Transform function for pretty printing options."""
+
+  def __init__(self, func, name):
+    self.f = func
+    self.name = name
+
+  def __repr__(self):
+    return '<' + self.name + '>'
+
+  def __call__(self, pil_img):
+    return self.f(pil_img)
+
+
+class TransformT(object):
+  """Each instance of this class represents a specific transform."""
+
+  def __init__(self, name, xform_fn):
+    self.name = name
+    self.xform = xform_fn
+
+  def pil_transformer(self, probability, level):
+
+    def return_function(im):
+      if random.random() < probability:
+        im = self.xform(im, level)
+      return im
+
+    name = self.name + '({:.1f},{})'.format(probability, level)
+    return TransformFunction(return_function, name)
+
+  def do_transform(self, image, level):
+    f = self.pil_transformer(PARAMETER_MAX, level)
+    return pil_unwrap(f(pil_wrap(image)))
+
+
+################## Transform Functions ##################
+identity = TransformT('identity', lambda pil_img, level: pil_img)
+flip_lr = TransformT(
+    'FlipLR',
+    lambda pil_img, level: pil_img.transpose(Image.FLIP_LEFT_RIGHT))
+flip_ud = TransformT(
+    'FlipUD',
+    lambda pil_img, level: pil_img.transpose(Image.FLIP_TOP_BOTTOM))
+# pylint:disable=g-long-lambda
+auto_contrast = TransformT(
+    'AutoContrast',
+    lambda pil_img, level: ImageOps.autocontrast(
+        pil_img.convert('RGB')).convert('RGBA'))
+equalize = TransformT(
+    'Equalize',
+    lambda pil_img, level: ImageOps.equalize(
+        pil_img.convert('RGB')).convert('RGBA'))
+invert = TransformT(
+    'Invert',
+    lambda pil_img, level: ImageOps.invert(
+        pil_img.convert('RGB')).convert('RGBA'))
+# pylint:enable=g-long-lambda
+blur = TransformT(
+    'Blur', lambda pil_img, level: pil_img.filter(ImageFilter.BLUR))
+smooth = TransformT(
+    'Smooth',
+    lambda pil_img, level: pil_img.filter(ImageFilter.SMOOTH))
+
+
+def _rotate_impl(pil_img, level):
+  """Rotates `pil_img` from -30 to 30 degrees depending on `level`."""
+  degrees = int_parameter(level, 30)
+  if random.random() > 0.5:
+    degrees = -degrees
+  return pil_img.rotate(degrees)
+
+
+rotate = TransformT('Rotate', _rotate_impl)
+
+
+def _posterize_impl(pil_img, level):
+  """Applies PIL Posterize to `pil_img`."""
+  level = int_parameter(level, 4)
+  return ImageOps.posterize(pil_img.convert('RGB'), 4 - level).convert('RGBA')
+
+
+posterize = TransformT('Posterize', _posterize_impl)
+
+
+def _shear_x_impl(pil_img, level):
+  """Applies PIL ShearX to `pil_img`.
+
+  The ShearX operation shears the image along the horizontal axis with `level`
+  magnitude.
+
+  Args:
+    pil_img: Image in PIL object.
+    level: Strength of the operation specified as an Integer from
+      [0, `PARAMETER_MAX`].
+
+  Returns:
+    A PIL Image that has had ShearX applied to it.
+  """
+  level = float_parameter(level, 0.3)
+  if random.random() > 0.5:
+    level = -level
+  return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), Image.AFFINE, (1, level, 0, 0, 1, 0))
+
+
+shear_x = TransformT('ShearX', _shear_x_impl)
+
+
+def _shear_y_impl(pil_img, level):
+  """Applies PIL ShearY to `pil_img`.
+
+  The ShearY operation shears the image along the vertical axis with `level`
+  magnitude.
+
+  Args:
+    pil_img: Image in PIL object.
+    level: Strength of the operation specified as an Integer from
+      [0, `PARAMETER_MAX`].
+
+  Returns:
+    A PIL Image that has had ShearX applied to it.
+  """
+  level = float_parameter(level, 0.3)
+  if random.random() > 0.5:
+    level = -level
+  return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), Image.AFFINE, (1, 0, 0, level, 1, 0))
+
+
+shear_y = TransformT('ShearY', _shear_y_impl)
+
+
+def _translate_x_impl(pil_img, level):
+  """Applies PIL TranslateX to `pil_img`.
+
+  Translate the image in the horizontal direction by `level`
+  number of pixels.
+
+  Args:
+    pil_img: Image in PIL object.
+    level: Strength of the operation specified as an Integer from
+      [0, `PARAMETER_MAX`].
+
+  Returns:
+    A PIL Image that has had TranslateX applied to it.
+  """
+  level = int_parameter(level, 10)
+  if random.random() > 0.5:
+    level = -level
+  return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), Image.AFFINE, (1, 0, level, 0, 1, 0))
+
+
+translate_x = TransformT('TranslateX', _translate_x_impl)
+
+
+def _translate_y_impl(pil_img, level):
+  """Applies PIL TranslateY to `pil_img`.
+
+  Translate the image in the vertical direction by `level`
+  number of pixels.
+
+  Args:
+    pil_img: Image in PIL object.
+    level: Strength of the operation specified as an Integer from
+      [0, `PARAMETER_MAX`].
+
+  Returns:
+    A PIL Image that has had TranslateY applied to it.
+  """
+  level = int_parameter(level, 10)
+  if random.random() > 0.5:
+    level = -level
+  return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), Image.AFFINE, (1, 0, 0, 0, 1, level))
+
+
+translate_y = TransformT('TranslateY', _translate_y_impl)
+
+
+def _crop_impl(pil_img, level, interpolation=Image.BILINEAR):
+  """Applies a crop to `pil_img` with the size depending on the `level`."""
+  cropped = pil_img.crop((level, level, IMAGE_SIZE - level, IMAGE_SIZE - level))
+  resized = cropped.resize((IMAGE_SIZE, IMAGE_SIZE), interpolation)
+  return resized
+
+
+crop_bilinear = TransformT('CropBilinear', _crop_impl)
+
+
+def _solarize_impl(pil_img, level):
+  """Applies PIL Solarize to `pil_img`.
+
+  Translate the image in the vertical direction by `level`
+  number of pixels.
+
+  Args:
+    pil_img: Image in PIL object.
+    level: Strength of the operation specified as an Integer from
+      [0, `PARAMETER_MAX`].
+
+  Returns:
+    A PIL Image that has had Solarize applied to it.
+  """
+  level = int_parameter(level, 256)
+  return ImageOps.solarize(pil_img.convert('RGB'), 256 - level).convert('RGBA')
+
+
+solarize = TransformT('Solarize', _solarize_impl)
+
+
+def _cutout_pil_impl(pil_img, level):
+  """Apply cutout to pil_img at the specified level."""
+  size = int_parameter(level, 20)
+  if size <= 0:
+    return pil_img
+  img_height, img_width, num_channels = (IMAGE_SIZE, IMAGE_SIZE, 3)
+  _, upper_coord, lower_coord = (
+      create_cutout_mask(img_height, img_width, num_channels, size))
+  pixels = pil_img.load()  # create the pixel map
+  for i in range(upper_coord[0], lower_coord[0]):  # for every col:
+    for j in range(upper_coord[1], lower_coord[1]):  # For every row
+      pixels[i, j] = (125, 122, 113, 0)  # set the colour accordingly
+  return pil_img
+
+cutout = TransformT('Cutout', _cutout_pil_impl)
+
+
+def _enhancer_impl(enhancer):
+  """Sets level to be between 0.1 and 1.8 for ImageEnhance transforms of PIL."""
+  def impl(pil_img, level):
+    v = float_parameter(level, 1.8) + .1  # going to 0 just destroys it
+    return enhancer(pil_img).enhance(v)
+  return impl
+
+
+color = TransformT('Color', _enhancer_impl(ImageEnhance.Color))
+contrast = TransformT('Contrast', _enhancer_impl(ImageEnhance.Contrast))
+brightness = TransformT('Brightness', _enhancer_impl(
+    ImageEnhance.Brightness))
+sharpness = TransformT('Sharpness', _enhancer_impl(ImageEnhance.Sharpness))
+
+ALL_TRANSFORMS = [
+    flip_lr,
+    flip_ud,
+    auto_contrast,
+    equalize,
+    invert,
+    rotate,
+    posterize,
+    crop_bilinear,
+    solarize,
+    color,
+    contrast,
+    brightness,
+    sharpness,
+    shear_x,
+    shear_y,
+    translate_x,
+    translate_y,
+    cutout,
+    blur,
+    smooth
+]
+
+NAME_TO_TRANSFORM = {t.name: t for t in ALL_TRANSFORMS}
+TRANSFORM_NAMES = NAME_TO_TRANSFORM.keys()

FAR-HO/blue_utils.py

@@ -0,0 +1,131 @@
+import matplotlib.pyplot as plt
+from far_ho.examples.datasets import Datasets, Dataset
+
+import os
+import numpy as np
+import tensorflow as tf
+
+import augmentation_transforms as augmentation_transforms ##### ATTENTION FICHIER EN DOUBLE => A REGLER MIEUX ####
+
+def viz_data(dataset, fig_name='data_sample',aug_policy=None):
+
+    plt.figure(figsize=(10,10))
+    for i in range(25):
+        plt.subplot(5,5,i+1)
+        plt.xticks([])
+        plt.yticks([])
+        plt.grid(False)
+
+        img = dataset.data[i][:,:,0]
+        if aug_policy :
+            img = augment_img(img,aug_policy)
+        #print('im shape',img.shape)
+        plt.imshow(img, cmap=plt.cm.binary)
+        plt.xlabel(np.nonzero(dataset.target[i])[0].item())
+
+    plt.savefig(fig_name)
+
+def augment_img(data, policy):
+
+    #print('Im shape',data.shape)
+    data = np.stack((data,)*3, axis=-1) #BOF BOF juste pour forcer 3 channels
+    #print('Im shape',data.shape)
+    final_img = augmentation_transforms.apply_policy(policy, data)
+    #final_img = augmentation_transforms.random_flip(augmentation_transforms.zero_pad_and_crop(final_img, 4))
+    # Apply cutout
+    #final_img = augmentation_transforms.cutout_numpy(final_img)
+    
+    im_rgb = np.array(final_img, np.float32)
+    im_gray = np.dot(im_rgb[...,:3], [0.2989, 0.5870, 0.1140]) #Just pour retourner a 1 channel
+
+    return im_gray
+
+
+### https://www.kaggle.com/raoulma/mnist-image-class-tensorflow-cnn-99-51-test-acc#5.-Build-the-neural-network-with-tensorflow-
+## build the neural network class
+# weight initialization
+def weight_variable(shape, name = None):
+    initial = tf.truncated_normal(shape, stddev=0.1)
+    return tf.Variable(initial, name = name)
+
+# bias initialization
+def bias_variable(shape, name = None):
+    initial = tf.constant(0.1, shape=shape) #  positive bias
+    return tf.Variable(initial, name = name)
+
+# 2D convolution
+def conv2d(x, W, name = None):
+    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME', name = name)
+
+# max pooling
+def max_pool_2x2(x, name = None):
+    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
+                              padding='SAME', name = name)
+
+def cnn(x_data_tf,y_data_tf, name='model'):
+     # tunable hyperparameters for nn architecture
+     s_f_conv1 = 3; # filter size of first convolution layer (default = 3)
+     n_f_conv1 = 36; # number of features of first convolution layer (default = 36)
+     s_f_conv2 = 3; # filter size of second convolution layer (default = 3)
+     n_f_conv2 = 36; # number of features of second convolution layer (default = 36)
+     s_f_conv3 = 3; # filter size of third convolution layer (default = 3)
+     n_f_conv3 = 36; # number of features of third convolution layer (default = 36)
+     n_n_fc1 = 576; # number of neurons of first fully connected layer (default = 576)
+
+     # 1.layer: convolution + max pooling
+     W_conv1_tf = weight_variable([s_f_conv1, s_f_conv1, 1, n_f_conv1], name = 'W_conv1_tf') # (5,5,1,32)
+     b_conv1_tf = bias_variable([n_f_conv1], name = 'b_conv1_tf') # (32)
+     h_conv1_tf = tf.nn.relu(conv2d(x_data_tf, 
+                                                 W_conv1_tf) + b_conv1_tf, 
+                                     name = 'h_conv1_tf') # (.,28,28,32)
+     h_pool1_tf = max_pool_2x2(h_conv1_tf, 
+                                            name = 'h_pool1_tf') # (.,14,14,32)
+
+     # 2.layer: convolution + max pooling
+     W_conv2_tf = weight_variable([s_f_conv2, s_f_conv2, 
+                                                n_f_conv1, n_f_conv2], 
+                                               name = 'W_conv2_tf')
+     b_conv2_tf = bias_variable([n_f_conv2], name = 'b_conv2_tf')
+     h_conv2_tf = tf.nn.relu(conv2d(h_pool1_tf, 
+                                                 W_conv2_tf) + b_conv2_tf, 
+                                     name ='h_conv2_tf') #(.,14,14,32)
+     h_pool2_tf = max_pool_2x2(h_conv2_tf, name = 'h_pool2_tf') #(.,7,7,32)
+
+     # 3.layer: convolution + max pooling
+     W_conv3_tf = weight_variable([s_f_conv3, s_f_conv3, 
+                                                n_f_conv2, n_f_conv3], 
+                                               name = 'W_conv3_tf')
+     b_conv3_tf = bias_variable([n_f_conv3], name = 'b_conv3_tf')
+     h_conv3_tf = tf.nn.relu(conv2d(h_pool2_tf, 
+                                                 W_conv3_tf) + b_conv3_tf, 
+                                     name = 'h_conv3_tf') #(.,7,7,32)
+     h_pool3_tf = max_pool_2x2(h_conv3_tf, 
+                                            name = 'h_pool3_tf') # (.,4,4,32)
+
+     # 4.layer: fully connected
+     W_fc1_tf = weight_variable([4*4*n_f_conv3,n_n_fc1], 
+                                             name = 'W_fc1_tf') # (4*4*32, 1024)
+     b_fc1_tf = bias_variable([n_n_fc1], name = 'b_fc1_tf') # (1024)
+     h_pool3_flat_tf = tf.reshape(h_pool3_tf, [-1,4*4*n_f_conv3], 
+                                          name = 'h_pool3_flat_tf') # (.,1024)
+     h_fc1_tf = tf.nn.relu(tf.matmul(h_pool3_flat_tf, 
+                                             W_fc1_tf) + b_fc1_tf, 
+                                   name = 'h_fc1_tf') # (.,1024)
+      
+     # add dropout
+     #keep_prob_tf = tf.placeholder(dtype=tf.float32, name = 'keep_prob_tf')
+     #h_fc1_drop_tf = tf.nn.dropout(h_fc1_tf, keep_prob_tf, name = 'h_fc1_drop_tf')
+
+     # 5.layer: fully connected
+     W_fc2_tf = weight_variable([n_n_fc1, 10], name = 'W_fc2_tf')
+     b_fc2_tf = bias_variable([10], name = 'b_fc2_tf')
+     z_pred_tf = tf.add(tf.matmul(h_fc1_tf, W_fc2_tf), 
+                                b_fc2_tf, name = 'z_pred_tf')# => (.,10)
+     # predicted probabilities in one-hot encoding
+     y_pred_proba_tf = tf.nn.softmax(z_pred_tf, name='y_pred_proba_tf') 
+        
+     # tensor of correct predictions
+     y_pred_correct_tf = tf.equal(tf.argmax(y_pred_proba_tf, 1),
+                                          tf.argmax(y_data_tf, 1),
+                                          name = 'y_pred_correct_tf')  
+     return y_pred_proba_tf

FAR-HO/far_pba_cifar.py

@@ -0,0 +1,166 @@
+#https://github.com/arcelien/pba/blob/master/autoaugment/train_cifar.py
+from __future__ import absolute_import, print_function, division
+
+import os
+import numpy as np
+import tensorflow as tf
+#import tensorflow.contrib.layers as layers
+import far_ho as far
+import far_ho.examples as far_ex
+#import pprint
+
+import autoaugment.augmentation_transforms as augmentation_transforms
+#import autoaugment.policies as found_policies
+from autoaugment.wrn import build_wrn_model
+
+
+def build_model(inputs, num_classes, is_training, hparams):
+  """Constructs the vision model being trained/evaled.
+  Args:
+    inputs: input features/images being fed to the image model build built.
+    num_classes: number of output classes being predicted.
+    is_training: is the model training or not.
+    hparams: additional hyperparameters associated with the image model.
+  Returns:
+    The logits of the image model.
+  """
+  scopes = setup_arg_scopes(is_training)
+  with contextlib.nested(*scopes):
+    if hparams.model_name == 'pyramid_net':
+      logits = build_shake_drop_model(
+          inputs, num_classes, is_training)
+    elif hparams.model_name == 'wrn':
+      logits = build_wrn_model(
+          inputs, num_classes, hparams.wrn_size)
+    elif hparams.model_name == 'shake_shake':
+      logits = build_shake_shake_model(
+          inputs, num_classes, hparams, is_training)
+  return logits
+
+
+class CifarModel(object):
+  """Builds an image model for Cifar10/Cifar100."""
+
+  def __init__(self, hparams):
+    self.hparams = hparams
+
+  def build(self, mode):
+    """Construct the cifar model."""
+    assert mode in ['train', 'eval']
+    self.mode = mode
+    self._setup_misc(mode)
+    self._setup_images_and_labels()
+    self._build_graph(self.images, self.labels, mode)
+
+    self.init = tf.group(tf.global_variables_initializer(),
+                         tf.local_variables_initializer())
+
+  def _setup_misc(self, mode):
+    """Sets up miscellaneous in the cifar model constructor."""
+    self.lr_rate_ph = tf.Variable(0.0, name='lrn_rate', trainable=False)
+    self.reuse = None if (mode == 'train') else True
+    self.batch_size = self.hparams.batch_size
+    if mode == 'eval':
+      self.batch_size = 25
+
+  def _setup_images_and_labels(self):
+    """Sets up image and label placeholders for the cifar model."""
+    if FLAGS.dataset == 'cifar10':
+      self.num_classes = 10
+    else:
+      self.num_classes = 100
+    self.images = tf.placeholder(tf.float32, [self.batch_size, 32, 32, 3])
+    self.labels = tf.placeholder(tf.float32,
+                                 [self.batch_size, self.num_classes])
+
+  def assign_epoch(self, session, epoch_value):
+    session.run(self._epoch_update, feed_dict={self._new_epoch: epoch_value})
+
+  def _build_graph(self, images, labels, mode):
+    """Constructs the TF graph for the cifar model.
+    Args:
+      images: A 4-D image Tensor
+      labels: A 2-D labels Tensor.
+      mode: string indicating training mode ( e.g., 'train', 'valid', 'test').
+    """
+    is_training = 'train' in mode
+    if is_training:
+      self.global_step = tf.train.get_or_create_global_step()
+
+    logits = build_model(
+        images,
+        self.num_classes,
+        is_training,
+        self.hparams)
+    self.predictions, self.cost = helper_utils.setup_loss(
+        logits, labels)
+    self.accuracy, self.eval_op = tf.metrics.accuracy(
+        tf.argmax(labels, 1), tf.argmax(self.predictions, 1))
+    self._calc_num_trainable_params()
+
+    # Adds L2 weight decay to the cost
+    self.cost = helper_utils.decay_weights(self.cost,
+                                           self.hparams.weight_decay_rate)
+ #### Attention: differe implem originale
+
+    self.init = tf.group(tf.global_variables_initializer(),
+                         tf.local_variables_initializer())
+
+
+########################################################
+
+######## PBA ############
+
+#Parallele Cifar model trainer
+tf.flags.DEFINE_string('model_name', 'wrn',
+                       'wrn, shake_shake_32, shake_shake_96, shake_shake_112, '
+                       'pyramid_net')
+tf.flags.DEFINE_string('checkpoint_dir', '/tmp/training', 'Training Directory.')
+tf.flags.DEFINE_string('data_path', '/tmp/data',
+                       'Directory where dataset is located.')
+tf.flags.DEFINE_string('dataset', 'cifar10',
+                       'Dataset to train with. Either cifar10 or cifar100')
+tf.flags.DEFINE_integer('use_cpu', 1, '1 if use CPU, else GPU.')
+## ???
+
+FLAGS = tf.flags.FLAGS
+FLAGS.dataset
+FLAGS.data_path
+FLAGS.model_name = 'wrn'
+
+hparams = tf.contrib.training.HParams(
+      train_size=50000,
+      validation_size=0,
+      eval_test=1,
+      dataset=FLAGS.dataset,
+      data_path=FLAGS.data_path,
+      batch_size=128,
+      gradient_clipping_by_global_norm=5.0)
+  if FLAGS.model_name == 'wrn':
+    hparams.add_hparam('model_name', 'wrn')
+    hparams.add_hparam('num_epochs', 200)
+    hparams.add_hparam('wrn_size', 160)
+    hparams.add_hparam('lr', 0.1)
+    hparams.add_hparam('weight_decay_rate', 5e-4)
+
+data_loader = data_utils.DataSet(hparams)
+data_loader.reset()
+
+with tf.Graph().as_default(): #, tf.device('/cpu:0' if FLAGS.use_cpu else '/gpu:0'):
+"""Builds the image models for train and eval."""
+    # Determine if we should build the train and eval model. When using
+    # distributed training we only want to build one or the other and not both.
+    with tf.variable_scope('model', use_resource=False):
+      m = CifarModel(self.hparams)
+      m.build('train')
+      #self._num_trainable_params = m.num_trainable_params
+      #self._saver = m.saver
+    #with tf.variable_scope('model', reuse=True, use_resource=False):
+    #  meval = CifarModel(self.hparams)
+    #  meval.build('eval')
+
+
+##### FAR-HO ####
+for _ in range(n_hyper_iterations):
+
+

FAR-HO/test.py

@@ -0,0 +1,92 @@
+import os
+import numpy as np
+import tensorflow as tf
+import tensorflow.contrib.layers as layers
+import far_ho as far
+import far_ho.examples as far_ex
+import matplotlib.pyplot as plt
+
+sess = tf.InteractiveSession()
+
+
+def get_data():
+    # load a small portion of mnist data
+    datasets = far_ex.mnist(data_root_folder=os.path.join(os.getcwd(), 'MNIST_DATA'), partitions=(.1, .1,))
+    return datasets.train, datasets.validation
+
+
+def g_logits(x,y):
+    with tf.variable_scope('model'):
+        h1 = layers.fully_connected(x, 300)
+        logits = layers.fully_connected(h1, int(y.shape[1]))
+    return logits
+
+
+x = tf.placeholder(tf.float32, shape=(None, 28**2), name='x')
+y = tf.placeholder(tf.float32, shape=(None, 10), name='y')
+logits = g_logits(x,y)
+train_set, validation_set = get_data()
+
+lambdas = far.get_hyperparameter('lambdas', tf.zeros(train_set.num_examples))
+lr = far.get_hyperparameter('lr', initializer=0.01)
+
+ce = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits)
+L = tf.reduce_mean(tf.sigmoid(lambdas)*ce)
+E = tf.reduce_mean(ce)
+
+accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)), tf.float32))
+
+inner_optimizer = far.GradientDescentOptimizer(lr)
+outer_optimizer = tf.train.AdamOptimizer()
+rev_it =10
+hyper_method = far.ReverseHG().truncated(reverse_iterations=rev_it)
+hyper_step = far.HyperOptimizer(hyper_method).minimize(E, outer_optimizer, L, inner_optimizer)
+
+T = 20  # Number of inner iterations
+train_set_supplier = train_set.create_supplier(x, y)
+validation_set_supplier = validation_set.create_supplier(x, y)
+tf.global_variables_initializer().run()
+
+print('inner:', L.eval(train_set_supplier()))
+print('outer:', E.eval(validation_set_supplier()))
+# print('-'*50)
+n_hyper_iterations = 200
+inner_losses = []
+outer_losses = []
+train_accs = []
+val_accs = []
+
+for _ in range(n_hyper_iterations):
+    hyper_step(T,
+               inner_objective_feed_dicts=train_set_supplier,
+               outer_objective_feed_dicts=validation_set_supplier)
+
+    inner_obj = L.eval(train_set_supplier())
+    outer_obj = E.eval(validation_set_supplier())
+    inner_losses.append(inner_obj)
+    outer_losses.append(outer_obj)
+    print('inner:', inner_obj)
+    print('outer:', outer_obj)
+
+    train_acc = accuracy.eval(train_set_supplier())
+    val_acc = accuracy.eval(validation_set_supplier())
+    train_accs.append(train_acc)
+    val_accs.append(val_acc)
+    print('training accuracy', train_acc)
+    print('validation accuracy', val_acc)
+
+    print('learning rate', lr.eval())
+    print('norm of examples weight', tf.norm(lambdas).eval())
+    print('-'*50)
+    
+plt.subplot(211)
+plt.plot(inner_losses, label='training loss')
+plt.plot(outer_losses, label='validation loss')
+plt.legend(loc=0, frameon=True)
+#plt.xlim(0, 19)
+plt.subplot(212)
+plt.plot(train_accs, label='training accuracy')
+plt.plot(val_accs, label='validation accuracy')
+plt.legend(loc=0, frameon=True)
+
+plt.savefig('H%d - I%d - R%d'%(n_hyper_iterations,T,rev_it))

FAR-HO/test_cnn.py

@@ -0,0 +1,126 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import os
+import numpy as np
+import tensorflow as tf
+import tensorflow.contrib.layers as layers
+import far_ho as far
+import far_ho.examples as far_ex
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+import matplotlib.pyplot as plt
+import blue_utils as butil
+
+#Reset
+try:
+    sess.close()
+except: pass
+rnd = np.random.RandomState(1)
+tf.reset_default_graph()
+sess = tf.InteractiveSession()
+
+def get_data(data_split):
+    # load a small portion of mnist data
+    datasets = far_ex.mnist(data_root_folder=os.path.join(os.getcwd(), 'MNIST_DATA'), partitions=data_split, reshape=False)
+    print("Data shape : ", datasets.train.dim_data, "/ Label shape : ", datasets.train.dim_target)
+    [print("Nb samples : ", d.num_examples) for d in datasets]
+    return datasets.train, datasets.validation, datasets.test
+
+#Model
+# FC : reshape = True
+def g_logits(x,y, name='model'):
+    with tf.variable_scope(name):
+        h1 = layers.fully_connected(x, 300)
+        logits = layers.fully_connected(h1, int(y.shape[1]))
+    return logits
+
+#### Hyper-parametres ####
+n_hyper_iterations = 500
+T = 20  # Number of inner iterations
+rev_it =10
+hp_lr = 1.e-3
+##########################
+
+#MNIST
+#x = tf.placeholder(tf.float32, shape=(None, 28**2), name='x')
+#y = tf.placeholder(tf.float32, shape=(None, 10), name='y')
+#logits = g_logits(x, y)
+
+#CNN : reshape = False
+x = tf.placeholder(dtype=tf.float32, shape=[None,28,28,1], name='x')
+y = tf.placeholder(dtype=tf.float32, shape=[None,10], name='y')
+
+logits = butil.cnn(x,y)
+
+train_set, validation_set, test_set = get_data(data_split=(.05, .05,))
+
+butil.viz_data(train_set)
+print('Data sampled !')
+
+# lambdas = far.get_hyperparameter('lambdas', tf.zeros(train_set.num_examples))
+#lr = far.get_hyperparameter('lr', initializer=1e-4, constraint=lambda t: tf.maximum(tf.minimum(t, .1), 1.e-7))
+#mu = far.get_hyperparameter('mu', initializer=0.9, constraint=lambda t: tf.maximum(tf.minimum(t, .99), 1.e-5))
+#rho = far.get_hyperparameter('rho', initializer=0.00001, constraint=lambda t: tf.maximum(tf.minimum(t, 0.01), 0.))
+lr = far.get_hyperparameter('lr', initializer=1e-4, constraint=lambda t: tf.maximum(tf.minimum(t, 1e-4), 1e-4))
+mu = far.get_hyperparameter('mu', initializer=0.9, constraint=lambda t: tf.maximum(tf.minimum(t, 0.9), 0.9))
+rho = far.get_hyperparameter('rho', initializer=0.00001, constraint=lambda t: tf.maximum(tf.minimum(t, 0.00001), 0.00001))
+
+ce = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits)
+L = tf.reduce_mean(ce) + rho*tf.add_n([tf.reduce_sum(w**2) for w in tf.trainable_variables()]) #Retirer la seconde partie de la loss quand HP inutiles
+E = tf.reduce_mean(ce)
+
+accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)), tf.float32))
+
+inner_optimizer = far.MomentumOptimizer(lr, mu)
+outer_optimizer = tf.train.AdamOptimizer(hp_lr)
+hyper_method = far.ReverseHG().truncated(reverse_iterations=rev_it)
+hyper_step = far.HyperOptimizer(hyper_method).minimize(E, outer_optimizer, L, inner_optimizer)
+
+train_set_supplier = train_set.create_supplier(x, y, batch_size=256)  # stochastic GD
+validation_set_supplier = validation_set.create_supplier(x, y)
+
+his_params = []
+
+tf.global_variables_initializer().run()
+
+for hyt in range(n_hyper_iterations):
+    hyper_step(T,
+               inner_objective_feed_dicts=train_set_supplier,
+               outer_objective_feed_dicts=validation_set_supplier)
+    res = sess.run(far.hyperparameters()) + [L.eval(train_set_supplier()), 
+                                             E.eval(validation_set_supplier()),
+                                             accuracy.eval(train_set_supplier()),
+                                             accuracy.eval(validation_set_supplier())]
+    his_params.append(res)
+
+    print('Hyper-it :',hyt,'/',n_hyper_iterations)
+    print('inner:', L.eval(train_set_supplier()))
+    print('outer:', E.eval(validation_set_supplier()))
+    print('training accuracy:', res[5])
+    print('validation accuracy:', res[6])
+    #print('learning rate', lr.eval(), 'momentum', mu.eval(), 'l2 coefficient', rho.eval())
+    print('-'*50)
+
+test_set_supplier = test_set.create_supplier(x, y)
+print('Test accuracy:',accuracy.eval(test_set_supplier()))
+
+fig, ax = plt.subplots(ncols=4, figsize=(15, 3))
+ax[0].set_title('Learning rate')
+ax[0].plot([e[0] for e in his_params])
+    
+ax[1].set_title('Momentum factor')
+ax[1].plot([e[1] for e in his_params]) 
+    
+#ax[2].set_title('L2 regulariz.')
+#ax[2].plot([e[2] for e in his_params])
+ax[2].set_title('Tr. and val. acc')
+ax[2].plot([e[5] for e in his_params])
+ax[2].plot([e[6] for e in his_params])
+    
+ax[3].set_title('Tr. and val. errors')
+ax[3].plot([e[3] for e in his_params])
+ax[3].plot([e[4] for e in his_params])  
+
+plt.savefig('res_cnn_H{}_I{}'.format(n_hyper_iterations,T))

FAR-HO/test_cnn_aug.py

@@ -0,0 +1,141 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import os
+import numpy as np
+import tensorflow as tf
+import tensorflow.contrib.layers as layers
+import far_ho as far
+import far_ho.examples as far_ex
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+import matplotlib.pyplot as plt
+import blue_utils as butil
+
+#Reset
+try:
+    sess.close()
+except: pass
+rnd = np.random.RandomState(1)
+tf.reset_default_graph()
+sess = tf.InteractiveSession()
+
+def get_data(data_split):
+    # load a small portion of mnist data
+    datasets = far_ex.mnist(data_root_folder=os.path.join(os.getcwd(), 'MNIST_DATA'), partitions=data_split, reshape=False)
+    print("Data shape : ", datasets.train.dim_data, "/ Label shape : ", datasets.train.dim_target)
+    [print("Nb samples : ", d.num_examples) for d in datasets]
+    return datasets.train, datasets.validation, datasets.test
+
+#Model
+# FC : reshape = True
+def g_logits(x,y, name='model'):
+    with tf.variable_scope(name):
+        h1 = layers.fully_connected(x, 300)
+        logits = layers.fully_connected(h1, int(y.shape[1]))
+    return logits
+
+#### Hyper-parametres ####
+n_hyper_iterations = 10
+T = 10 # Number of inner iterations
+rev_it =10
+hp_lr = 0.02
+##########################
+
+#MNIST
+#x = tf.placeholder(tf.float32, shape=(None, 28**2), name='x')
+#y = tf.placeholder(tf.float32, shape=(None, 10), name='y')
+#logits = g_logits(x, y)
+
+#CNN : reshape = False
+x = tf.placeholder(dtype=tf.float32, shape=[None,28,28,1], name='x')
+y = tf.placeholder(dtype=tf.float32, shape=[None,10], name='y')
+
+logits = butil.cnn(x,y)
+
+train_set, validation_set, test_set = get_data(data_split=(.1, .1,))
+
+probX = far.get_hyperparameter('probX', initializer=0.1, constraint=lambda t: tf.maximum(tf.minimum(t, 0.1), 0.9))
+probY = far.get_hyperparameter('probY', initializer=0.1, constraint=lambda t: tf.maximum(tf.minimum(t, 0.1), 0.9))
+
+#lr = far.get_hyperparameter('lr', initializer=1e-4, constraint=lambda t: tf.maximum(tf.minimum(t, 1e-4), 1e-4))
+#mu = far.get_hyperparameter('mu', initializer=0.9, constraint=lambda t: tf.maximum(tf.minimum(t, 0.9), 0.9))
+
+#probX, probY = 0.5, 0.5
+#policy = [('TranslateX', probX, 8), ('TranslateY', probY, 8)]
+policy = [('TranslateX', probX, 8), ('FlipUD', probY, 8)]
+print('Hyp :',far.utils.hyperparameters(scope=None))
+
+#butil.viz_data(train_set, aug_policy= policy)
+#print('Data sampled !')
+
+#Ajout artificiel des transfo a la loss juste pour qu il soit compter dans la dynamique du graph
+probX_loss = tf.sigmoid(probX)
+probY_loss = tf.sigmoid(probY)
+
+ce = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits)
+L = tf.reduce_mean(probX_loss*probY_loss*ce)
+E = tf.reduce_mean(ce)
+
+accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)), tf.float32))
+
+inner_optimizer = far.AdamOptimizer()
+outer_optimizer = tf.train.AdamOptimizer(hp_lr)
+hyper_method = far.ReverseHG().truncated(reverse_iterations=rev_it)
+hyper_step = far.HyperOptimizer(hyper_method).minimize(E, outer_optimizer, L, inner_optimizer)
+
+train_set_supplier = train_set.create_supplier(x, y, batch_size=256, aug_policy=policy)  # stochastic GD
+validation_set_supplier = validation_set.create_supplier(x, y)
+
+#print(train_set.dim_data,validation_set.dim_data)
+
+his_params = []
+
+tf.global_variables_initializer().run()
+
+butil.viz_data(train_set, fig_name= 'Start_sample',aug_policy= policy)
+print('Data sampled !')
+
+for hyt in range(n_hyper_iterations):
+    hyper_step(T,
+               inner_objective_feed_dicts=train_set_supplier,
+               outer_objective_feed_dicts=validation_set_supplier,
+               _skip_hyper_ts=True)
+    res = sess.run(far.hyperparameters()) + [L.eval(train_set_supplier()), 
+                                             E.eval(validation_set_supplier()),
+                                             accuracy.eval(train_set_supplier()),
+                                             accuracy.eval(validation_set_supplier())]
+    his_params.append(res)
+
+    butil.viz_data(train_set, fig_name= 'Train_sample_{}'.format(hyt),aug_policy= policy)
+    print('Data sampled !')
+
+    print('Hyper-it :',hyt,'/',n_hyper_iterations)
+    print('inner:', L.eval(train_set_supplier()))
+    print('outer:', E.eval(validation_set_supplier()))
+    print('training accuracy:', res[4])
+    print('validation accuracy:', res[5])
+    print('Transformation : ProbX -',res[0],'/ProbY -',res[1])
+    #print('learning rate', lr.eval(), 'momentum', mu.eval(), 'l2 coefficient', rho.eval())
+    print('-'*50)
+
+test_set_supplier = test_set.create_supplier(x, y)
+print('Test accuracy:',accuracy.eval(test_set_supplier()))
+
+fig, ax = plt.subplots(ncols=4, figsize=(15, 3))
+ax[0].set_title('ProbX')
+ax[0].plot([e[0] for e in his_params])
+    
+ax[1].set_title('ProbY')
+ax[1].plot([e[1] for e in his_params]) 
+    
+ax[2].set_title('Tr. and val. errors')
+ax[2].plot([e[2] for e in his_params])
+ax[2].plot([e[3] for e in his_params])  
+
+ax[3].set_title('Tr. and val. acc')
+ax[3].plot([e[4] for e in his_params])
+ax[3].plot([e[5] for e in his_params])
+
+plt.savefig('res_cnn_aug_H{}_I{}'.format(n_hyper_iterations,T))

FAR-HO/test_fc.py

@@ -0,0 +1,133 @@
+#https://github.com/lucfra/FAR-HO/blob/master/far_ho/examples/autoMLDemos/Far-HO%20Demo%2C%20AutoML%202018%2C%20ICML%20workshop.ipynb
+import warnings
+warnings.filterwarnings("ignore")
+
+import os
+import numpy as np
+import tensorflow as tf
+import tensorflow.contrib.layers as layers
+import far_ho as far
+import far_ho.examples as far_ex
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+import matplotlib.pyplot as plt
+#import blue_utils as butil
+
+#Reset
+try:
+    sess.close()
+except: pass
+rnd = np.random.RandomState(1)
+tf.reset_default_graph()
+sess = tf.InteractiveSession()
+
+def get_data(data_split):
+    # load a small portion of mnist data
+    datasets = far_ex.mnist(data_root_folder=os.path.join(os.getcwd(), 'MNIST_DATA'), partitions=data_split, reshape=True)
+    print("Data shape : ", datasets.train.dim_data, " / Label shape : ", datasets.train.dim_target)
+    [print("Nb samples : ", d.num_examples) for d in datasets]
+    return datasets.train, datasets.validation, datasets.test
+
+#Model
+# FC : reshape = True
+def g_logits(x,y, name='model'):
+    with tf.variable_scope(name):
+        h1 = layers.fully_connected(x, 300)
+        logits = layers.fully_connected(h1, int(y.shape[1]))
+    return logits
+
+#### Hyper-parametres ####
+n_hyper_iterations = 90
+T = 20  # Number of inner iterations
+rev_it =10
+hp_lr = 0.1
+epochs =10
+batch_size = 256
+##########################
+
+#MNIST
+x = tf.placeholder(tf.float32, shape=(None, 28**2), name='x')
+y = tf.placeholder(tf.float32, shape=(None, 10), name='y')
+logits = g_logits(x, y)
+
+#CNN : reshape = False
+#x = tf.placeholder(dtype=tf.float32, shape=[None,28,28,1], name='x')
+#y = tf.placeholder(dtype=tf.float32, shape=[None,10], name='y')
+
+#logits = butil.cnn(x,y)
+
+train_set, validation_set, test_set = get_data(data_split=(.6, .3,))
+
+#butil.viz_data(train_set)
+
+# lambdas = far.get_hyperparameter('lambdas', tf.zeros(train_set.num_examples))
+lr = far.get_hyperparameter('lr', initializer=1e-2, constraint=lambda t: tf.maximum(tf.minimum(t, 0.1), 1.e-7))
+mu = far.get_hyperparameter('mu', initializer=0.95, constraint=lambda t: tf.maximum(tf.minimum(t, .99), 1.e-5))
+#rho = far.get_hyperparameter('rho', initializer=0.00001, constraint=lambda t: tf.maximum(tf.minimum(t, 0.01), 0.))
+
+
+ce = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits)
+L = tf.reduce_mean(ce) #+ rho*tf.add_n([tf.reduce_sum(w**2) for w in tf.trainable_variables()]) #Retirer la seconde partie de la loss quand HP inutiles
+E = tf.reduce_mean(ce)
+
+accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)), tf.float32))
+
+inner_optimizer = far.MomentumOptimizer(lr, mu)
+#inner_optimizer = far.GradientDescentOptimizer(lr)
+outer_optimizer = tf.train.AdamOptimizer(hp_lr)
+hyper_method = far.ReverseHG().truncated(reverse_iterations=rev_it)
+hyper_step = far.HyperOptimizer(hyper_method).minimize(E, outer_optimizer, L, inner_optimizer)#, global_step=tf.train.get_or_create_step())
+
+train_set_supplier = train_set.create_supplier(x, y, batch_size=batch_size)#, epochs=1)  # stochastic GD
+validation_set_supplier = validation_set.create_supplier(x, y)
+
+
+print('Hyper iterations par epochs',int(train_set.num_examples/batch_size*epochs/T))
+
+his_params = []
+
+tf.global_variables_initializer().run()
+
+for hyt in range(n_hyper_iterations):
+    hyper_step(T,
+               inner_objective_feed_dicts=train_set_supplier,
+               outer_objective_feed_dicts=validation_set_supplier,
+               _skip_hyper_ts=False)
+    res = sess.run(far.hyperparameters()) + [0, L.eval(train_set_supplier()), 
+                                             E.eval(validation_set_supplier()),
+                                             accuracy.eval(train_set_supplier()),
+                                             accuracy.eval(validation_set_supplier())]
+
+    his_params.append(res)
+
+    print('Hyper-it :',hyt,'/',n_hyper_iterations)
+    print('inner:', res[3])
+    print('outer:', res[4])
+    print('training accuracy:', res[5])
+    print('validation accuracy:', res[6])
+    #print('learning rate', lr.eval(), 'momentum', mu.eval(), 'l2 coefficient', rho.eval())
+    print('-'*50)
+
+test_set_supplier = test_set.create_supplier(x, y)
+print('Test accuracy:',accuracy.eval(test_set_supplier()))
+
+fig, ax = plt.subplots(ncols=4, figsize=(15, 3))
+ax[0].set_title('Learning rate')
+ax[0].plot([e[0] for e in his_params])
+    
+ax[1].set_title('Momentum factor')
+ax[1].plot([e[1] for e in his_params]) 
+    
+#ax[2].set_title('L2 regulariz.')
+#ax[2].plot([e[2] for e in his_params])
+ax[2].set_title('Tr. and val. acc')
+ax[2].plot([e[5] for e in his_params])
+ax[2].plot([e[6] for e in his_params])
+    
+ax[3].set_title('Tr. and val. errors')
+ax[3].plot([e[3] for e in his_params])
+ax[3].plot([e[4] for e in his_params])  
+
+plt.savefig('resultats/res_fc_H{}_I{}'.format(n_hyper_iterations,T))
+#plt.savefig('resultats/res_fc_H{}_I{}_noHyp'.format(n_hyper_iterations,T))