Add unsupervised learning experiments for linear_ntk

finite_ntk/README.md

@@ -1,8 +1,6 @@
 # Transfer Learning via Linearized Neural Networks
 
-This repository contains a GPyTorch implementation of finite width neural tangent kernels from the paper [(link)](https://arxiv.org/abs/2103.01439) 
-
-*Fast Adaptation with Linearized Neural Networks*
+This repository contains a GPyTorch implementation of finite width neural tangent kernels from the paper [Fast Adaptation with Linearized Neural Networks](https://arxiv.org/abs/2103.01439) 
 
 by Wesley Maddox, Shuai Tang, Pablo Garcia Moreno, Andrew Gordon Wilson, and Andreas Damianou,
 

finite_ntk/experiments/README.md

@@ -0,0 +1,8 @@
+
+
+## Olivetti
+
+First, run `cd dataset; python adaptation_dataset_maker.py` to download Olivetti and construct the dataset.
+
+Then run `python run_ntk.py --prop=XXX` (note that the Fisher flag is untested.)
+the defaults are the learning rates/etc. we used for your proportion.

finite_ntk/experiments/dataset/adaptation_dataset_maker.py

@@ -0,0 +1,76 @@
+import math
+import numpy as np
+import sklearn.datasets as datasets
+from rotators import *
+
+def main():
+    olivetti = datasets.fetch_olivetti_faces()
+    inputs = olivetti['data']
+    images = olivetti['images']
+    targets = olivetti['target']
+
+    repeat_images = 30 # the number of times we want to repeat each image
+    n_images = images.shape[0] * repeat_images
+    image_sz = images.shape[1]
+
+    image_keep = int(math.sqrt(2) * image_sz/2)
+
+    rotated_faces = np.zeros((n_images, image_keep, image_keep))
+    all_targets = np.zeros(n_images)
+    angles = np.pi * np.random.rand(n_images) - np.pi/2
+    degrees = 180/np.pi * angles
+
+    img_ind = 0
+    for rpt in range(repeat_images):
+        for face_ind in range(images.shape[0]):
+            rotated_image = rotate_image(images[face_ind, :, :], degrees[img_ind])
+            cropped_image = crop_around_center(rotated_image, image_keep, image_keep)
+            all_targets[img_ind] = targets[face_ind]
+
+            rotated_faces[img_ind, :, :] = cropped_image
+            img_ind += 1
+
+    ## separate training data out ##
+    n_people = 40
+
+    n_train_people = 20
+
+    ## randomly select the 20 people for training and 10 for test
+    shuffle_people = np.random.permutation(n_people)
+    train_people = shuffle_people[:n_train_people]
+    test_people = shuffle_people[n_train_people:]
+
+    train_images = np.zeros((1, image_keep, image_keep))
+    train_angles = np.zeros((1))
+    for tp in train_people:
+        keepers = np.where(all_targets == tp)[0]
+        keep_imgs = rotated_faces[np.ix_(keepers), :, :].squeeze()
+        keep_angles = degrees[np.ix_(keepers)]
+        train_images = np.concatenate((train_images, keep_imgs), 0)
+        train_angles = np.concatenate((train_angles, keep_angles))
+
+    train_images = np.expand_dims(train_images[1:, :, :], 1)
+    train_angles = train_angles[1:]
+
+
+    test_images = np.zeros((1, image_keep, image_keep))
+    test_angles = np.zeros((1))
+    test_people_ids = np.zeros((1))
+    for i, tp in enumerate(test_people):
+        keepers = np.where(all_targets == tp)[0]
+        keep_imgs = rotated_faces[np.ix_(keepers), :, :].squeeze()
+        keep_angles = degrees[np.ix_(keepers)]
+        test_images = np.concatenate((test_images, keep_imgs), 0)
+        test_angles = np.concatenate((test_angles, keep_angles))
+        test_people_ids = np.concatenate((test_people_ids, i* np.ones(keep_imgs.shape[0])))
+
+    test_images = np.expand_dims(test_images[1:, :, :], 1)
+    test_angles = test_angles[1:]
+    test_people_ids = test_people_ids[1:]
+
+    np.savez("./rotated_faces_data_withids.npz",
+             train_images=train_images, train_angles=train_angles,
+             test_images=test_images, test_angles=test_angles, test_people_ids=test_people_ids)
+
+if __name__ == '__main__':
+    main()

finite_ntk/experiments/dataset/get_faces_loaders.py

@@ -0,0 +1,28 @@
+import torch
+import numpy as np
+
+
+def get_faces_loaders(batch_size=128, test=True, data_path="./data/"):
+    """
+    returns the train (and test if selected) loaders for the olivetti
+    rotated faces dataset
+    """
+
+    dat = np.load(data_path + "rotated_faces_data.npz")
+    train_images = torch.FloatTensor(dat['train_images'])
+    train_targets = torch.FloatTensor(dat['train_angles'])
+
+    traindata = torch.utils.data.TensorDataset(train_images, train_targets)
+    trainloader = torch.utils.data.DataLoader(traindata, batch_size=batch_size,
+                                              shuffle=True)
+
+    if test:
+        test_images = torch.FloatTensor(dat['test_images'])
+        test_targets = torch.FloatTensor(dat['test_angles'])
+
+        testdata = torch.utils.data.TensorDataset(test_images, test_targets)
+        testloader = torch.utils.data.DataLoader(testdata, batch_size=batch_size)
+
+        return trainloader, testloader
+
+    return trainloader

finite_ntk/experiments/dataset/rotators.py

@@ -0,0 +1,130 @@
+import math
+import cv2
+import numpy as np
+
+def rotate_image(image, angle):
+    """
+    Rotates an OpenCV 2 / NumPy image about it's centre by the given angle
+    (in degrees). The returned image will be large enough to hold the entire
+    new image, with a black background
+    """
+
+    # Get the image size
+    # No that's not an error - NumPy stores image matricies backwards
+    image_size = (image.shape[1], image.shape[0])
+    image_center = tuple(np.array(image_size) / 2)
+
+    # Convert the OpenCV 3x2 rotation matrix to 3x3
+    rot_mat = np.vstack(
+        [cv2.getRotationMatrix2D(image_center, angle, 1.0), [0, 0, 1]]
+    )
+
+    rot_mat_notranslate = np.matrix(rot_mat[0:2, 0:2])
+
+    # Shorthand for below calcs
+    image_w2 = image_size[0] * 0.5
+    image_h2 = image_size[1] * 0.5
+
+    # Obtain the rotated coordinates of the image corners
+    rotated_coords = [
+        (np.array([-image_w2,  image_h2]) * rot_mat_notranslate).A[0],
+        (np.array([ image_w2,  image_h2]) * rot_mat_notranslate).A[0],
+        (np.array([-image_w2, -image_h2]) * rot_mat_notranslate).A[0],
+        (np.array([ image_w2, -image_h2]) * rot_mat_notranslate).A[0]
+    ]
+
+    # Find the size of the new image
+    x_coords = [pt[0] for pt in rotated_coords]
+    x_pos = [x for x in x_coords if x > 0]
+    x_neg = [x for x in x_coords if x < 0]
+
+    y_coords = [pt[1] for pt in rotated_coords]
+    y_pos = [y for y in y_coords if y > 0]
+    y_neg = [y for y in y_coords if y < 0]
+
+    right_bound = max(x_pos)
+    left_bound = min(x_neg)
+    top_bound = max(y_pos)
+    bot_bound = min(y_neg)
+
+    new_w = int(abs(right_bound - left_bound))
+    new_h = int(abs(top_bound - bot_bound))
+
+    # We require a translation matrix to keep the image centred
+    trans_mat = np.matrix([
+        [1, 0, int(new_w * 0.5 - image_w2)],
+        [0, 1, int(new_h * 0.5 - image_h2)],
+        [0, 0, 1]
+    ])
+
+    # Compute the tranform for the combined rotation and translation
+    affine_mat = (np.matrix(trans_mat) * np.matrix(rot_mat))[0:2, :]
+
+    # Apply the transform
+    result = cv2.warpAffine(
+        image,
+        affine_mat,
+        (new_w, new_h),
+        flags=cv2.INTER_LINEAR
+    )
+
+    return result
+
+
+def largest_rotated_rect(w, h, angle):
+    """
+    Given a rectangle of size wxh that has been rotated by 'angle' (in
+    radians), computes the width and height of the largest possible
+    axis-aligned rectangle within the rotated rectangle.
+
+    Original JS code by 'Andri' and Magnus Hoff from Stack Overflow
+
+    Converted to Python by Aaron Snoswell
+    """
+
+    quadrant = int(math.floor(angle / (math.pi / 2))) & 3
+    sign_alpha = angle if ((quadrant & 1) == 0) else math.pi - angle
+    alpha = (sign_alpha % math.pi + math.pi) % math.pi
+
+    bb_w = w * math.cos(alpha) + h * math.sin(alpha)
+    bb_h = w * math.sin(alpha) + h * math.cos(alpha)
+
+    gamma = math.atan2(bb_w, bb_w) if (w < h) else math.atan2(bb_w, bb_w)
+
+    delta = math.pi - alpha - gamma
+
+    length = h if (w < h) else w
+
+    d = length * math.cos(alpha)
+    a = d * math.sin(alpha) / math.sin(delta)
+
+    y = a * math.cos(gamma)
+    x = y * math.tan(gamma)
+
+    return (
+        bb_w - 2 * x,
+        bb_h - 2 * y
+    )
+
+
+def crop_around_center(image, width, height):
+    """
+    Given a NumPy / OpenCV 2 image, crops it to the given width and height,
+    around it's centre point
+    """
+
+    image_size = (image.shape[1], image.shape[0])
+    image_center = (int(image_size[0] * 0.5), int(image_size[1] * 0.5))
+
+    if(width > image_size[0]):
+        width = image_size[0]
+
+    if(height > image_size[1]):
+        height = image_size[1]
+
+    x1 = int(image_center[0] - width * 0.5)
+    x2 = int(image_center[0] + width * 0.5)
+    y1 = int(image_center[1] - height * 0.5)
+    y2 = int(image_center[1] + height * 0.5)
+
+    return image[y1:y2, x1:x2]

finite_ntk/experiments/olivetti/README.md

@@ -0,0 +1,8 @@
+
+
+## Olivetti
+
+First, run `cd dataset; python adaptation_dataset_maker.py` to download Olivetti and construct the dataset.
+
+Then run `python run_ntk.py --prop=XXX` (note that the Fisher flag is untested.)
+the defaults are the learning rates/etc. we used for your proportion.

finite_ntk/experiments/olivetti/dataset/adaptation_dataset_maker.py

@@ -0,0 +1,90 @@
+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License").
+#   You may not use this file except in compliance with the License.
+#   A copy of the License is located at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   or in the "license" file accompanying this file. This file is distributed
+#   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#   express or implied. See the License for the specific language governing
+#   permissions and limitations under the License.
+# ==============================================================================
+
+import math
+import numpy as np
+import sklearn.datasets as datasets
+from rotators import *
+
+def main():
+    olivetti = datasets.fetch_olivetti_faces()
+    inputs = olivetti['data']
+    images = olivetti['images']
+    targets = olivetti['target']
+
+    repeat_images = 30 # the number of times we want to repeat each image
+    n_images = images.shape[0] * repeat_images
+    image_sz = images.shape[1]
+
+    image_keep = int(math.sqrt(2) * image_sz/2)
+
+    rotated_faces = np.zeros((n_images, image_keep, image_keep))
+    all_targets = np.zeros(n_images)
+    angles = np.pi * np.random.rand(n_images) - np.pi/2
+    degrees = 180/np.pi * angles
+
+    img_ind = 0
+    for rpt in range(repeat_images):
+        for face_ind in range(images.shape[0]):
+            rotated_image = rotate_image(images[face_ind, :, :], degrees[img_ind])
+            cropped_image = crop_around_center(rotated_image, image_keep, image_keep)
+            all_targets[img_ind] = targets[face_ind]
+
+            rotated_faces[img_ind, :, :] = cropped_image
+            img_ind += 1
+
+    ## separate training data out ##
+    n_people = 40
+
+    n_train_people = 20
+
+    ## randomly select the 20 people for training and 10 for test
+    shuffle_people = np.random.permutation(n_people)
+    train_people = shuffle_people[:n_train_people]
+    test_people = shuffle_people[n_train_people:]
+
+    train_images = np.zeros((1, image_keep, image_keep))
+    train_angles = np.zeros((1))
+    for tp in train_people:
+        keepers = np.where(all_targets == tp)[0]
+        keep_imgs = rotated_faces[np.ix_(keepers), :, :].squeeze()
+        keep_angles = degrees[np.ix_(keepers)]
+        train_images = np.concatenate((train_images, keep_imgs), 0)
+        train_angles = np.concatenate((train_angles, keep_angles))
+
+    train_images = np.expand_dims(train_images[1:, :, :], 1)
+    train_angles = train_angles[1:]
+
+
+    test_images = np.zeros((1, image_keep, image_keep))
+    test_angles = np.zeros((1))
+    test_people_ids = np.zeros((1))
+    for i, tp in enumerate(test_people):
+        keepers = np.where(all_targets == tp)[0]
+        keep_imgs = rotated_faces[np.ix_(keepers), :, :].squeeze()
+        keep_angles = degrees[np.ix_(keepers)]
+        test_images = np.concatenate((test_images, keep_imgs), 0)
+        test_angles = np.concatenate((test_angles, keep_angles))
+        test_people_ids = np.concatenate((test_people_ids, i* np.ones(keep_imgs.shape[0])))
+
+    test_images = np.expand_dims(test_images[1:, :, :], 1)
+    test_angles = test_angles[1:]
+    test_people_ids = test_people_ids[1:]
+
+    np.savez("./rotated_faces_data_withids.npz",
+             train_images=train_images, train_angles=train_angles,
+             test_images=test_images, test_angles=test_angles, test_people_ids=test_people_ids)
+
+if __name__ == '__main__':
+    main()

finite_ntk/experiments/olivetti/dataset/get_faces_loaders.py

@@ -0,0 +1,42 @@
+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License").
+#   You may not use this file except in compliance with the License.
+#   A copy of the License is located at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   or in the "license" file accompanying this file. This file is distributed
+#   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#   express or implied. See the License for the specific language governing
+#   permissions and limitations under the License.
+# ==============================================================================
+
+import torch
+import numpy as np
+
+
+def get_faces_loaders(batch_size=128, test=True, data_path="./data/"):
+    """
+    returns the train (and test if selected) loaders for the olivetti
+    rotated faces dataset
+    """
+
+    dat = np.load(data_path + "rotated_faces_data.npz")
+    train_images = torch.FloatTensor(dat['train_images'])
+    train_targets = torch.FloatTensor(dat['train_angles'])
+
+    traindata = torch.utils.data.TensorDataset(train_images, train_targets)
+    trainloader = torch.utils.data.DataLoader(traindata, batch_size=batch_size,
+                                              shuffle=True)
+
+    if test:
+        test_images = torch.FloatTensor(dat['test_images'])
+        test_targets = torch.FloatTensor(dat['test_angles'])
+
+        testdata = torch.utils.data.TensorDataset(test_images, test_targets)
+        testloader = torch.utils.data.DataLoader(testdata, batch_size=batch_size)
+
+        return trainloader, testloader
+
+    return trainloader