Add implementations, implement S3SwapStore (with only tars), FSRepoStore, Tar and File utils, add tests

services/git-bridge/pom.xml

@@ -140,5 +140,29 @@
             <artifactId>mockito-core</artifactId>
             <version>1.10.19</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk</artifactId>
+            <version>1.11.28</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.5</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.12</version>
+        </dependency>
     </dependencies>
 </project>

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java

@@ -30,6 +30,7 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
 import uk.ac.ic.wlgitbridge.util.Log;
 
 import java.io.IOException;
+import java.time.Duration;
 import java.util.*;
 
 /**
@@ -104,8 +105,8 @@ public class Bridge {
         Log.info("Bye");
     }
 
-    public void startSwapJob(int intervalMillis) {
-        swapJob.start(intervalMillis);
+    public void startSwapJob(Duration interval) {
+        swapJob.start(interval);
     }
 
     /* TODO: Remove these when WLBridged is moved into RepoStore */

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java

@@ -1,5 +1,6 @@
 package uk.ac.ic.wlgitbridge.bridge.db;
 
+import java.sql.Timestamp;
 import java.util.List;
 
 /**
@@ -19,4 +20,13 @@ public interface DBStore {
 
     String getPathForURLInProject(String projectName, String url);
 
+    String getOldestUnswappedProject();
+
+    /**
+     * Sets the last accessed time for the given project name.
+     * @param projectName the project's name
+     * @param time the time, or null if the project is to be swapped
+     */
+    void setLastAccessedTime(String projectName, Timestamp time);
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java

@@ -5,6 +5,7 @@ import uk.ac.ic.wlgitbridge.util.Log;
 
 import java.io.File;
 import java.sql.SQLException;
+import java.sql.Timestamp;
 import java.util.Arrays;
 import java.util.List;
 
@@ -84,4 +85,14 @@ public class SqliteDBStore implements DBStore {
         }
     }
 
+    @Override
+    public String getOldestUnswappedProject() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void setLastAccessedTime(String projectName, Timestamp time) {
+        throw new UnsupportedOperationException();
+    }
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java

@@ -1,12 +1,20 @@
 package uk.ac.ic.wlgitbridge.bridge.repo;
 
-import uk.ac.ic.wlgitbridge.util.Util;
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import uk.ac.ic.wlgitbridge.util.Project;
+import uk.ac.ic.wlgitbridge.util.Tar;
 
 import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
+import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
+
 /**
  * Created by winston on 20/08/2016.
  */
@@ -30,6 +38,7 @@ public class FSRepoStore implements RepoStore {
         return rootDirectory;
     }
 
+    /* TODO: Perhaps we should just delete bad directories on the fly. */
     @Override
     public void purgeNonexistentProjects(
             Collection<String> existingProjectNames
@@ -37,15 +46,63 @@ public class FSRepoStore implements RepoStore {
         List<String> excludedFromDeletion =
                 new ArrayList<>(existingProjectNames);
         excludedFromDeletion.add(".wlgb");
-        Util.deleteInDirectoryApartFrom(
+        deleteInDirectoryApartFrom(
                 rootDirectory,
                 excludedFromDeletion.toArray(new String[] {})
         );
     }
 
+    @Override
+    public long totalSize() {
+        return FileUtils.sizeOfDirectory(rootDirectory);
+    }
+
+    @Override
+    public InputStream bzip2Project(String projectName) throws IOException {
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        return Tar.tar(getDotGitForProject(projectName));
+    }
+
+    @Override
+    public void remove(String projectName) throws IOException {
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        FileUtils.deleteDirectory(new File(rootDirectory, projectName));
+    }
+
+    @Override
+    public void unbzip2Project(
+            String projectName,
+            InputStream dataStream
+    ) throws IOException {
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        Preconditions.checkState(getDirForProject(projectName).mkdirs());
+        Tar.untar(dataStream, getDirForProject(projectName));
+    }
+
+    private File getDirForProject(String projectName) {
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        return Paths.get(
+                rootDirectory.getAbsolutePath()
+        ).resolve(
+                projectName
+        ).toFile();
+    }
+
+    private File getDotGitForProject(String projectName) {
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        return Paths.get(
+                rootDirectory.getAbsolutePath()
+        ).resolve(
+                projectName
+        ).resolve(
+                ".git"
+        ).toFile();
+    }
+
     private File initRootGitDirectory(String rootGitDirectoryPath) {
         File rootGitDirectory = new File(rootGitDirectoryPath);
         rootGitDirectory.mkdirs();
+        Preconditions.checkArgument(rootGitDirectory.isDirectory());
         return rootGitDirectory;
     }
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java

@@ -1,8 +1,8 @@
 package uk.ac.ic.wlgitbridge.bridge.repo;
 
-import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
-
 import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Collection;
 
 /**
@@ -14,9 +14,28 @@ public interface RepoStore {
 
     File getRootDirectory();
 
-
     void purgeNonexistentProjects(
             Collection<String> existingProjectNames
     );
 
+    long totalSize();
+
+    /*
+     * Tars and bzip2s the .git directory of the given project. Throws an
+     * IOException if the project doesn't exist. The returned stream is a copy
+     * of the original .git directory, which must be deleted using remove().
+     */
+    InputStream bzip2Project(String projectName) throws IOException;
+
+    void remove(String projectName) throws IOException;
+
+    /**
+     * Unbzip2s the given data stream into a .git directory for projectName.
+     * Creates the project directory.
+     * If projectName already exists, throws an IOException.
+     * @param projectName the name of the project, e.g. abc123
+     * @param dataStream the data stream containing the bzipped contents.
+     */
+    void unbzip2Project(String projectName, InputStream dataStream) throws IOException;
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java

@@ -0,0 +1,64 @@
+package uk.ac.ic.wlgitbridge.bridge.swap;
+
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.*;
+
+import java.io.InputStream;
+
+/**
+ * Created by winston on 21/08/2016.
+ */
+public class S3SwapStore implements SwapStore {
+
+    private final AmazonS3 s3;
+
+    private final String bucketName;
+
+    public S3SwapStore(
+            String accessKey,
+            String secret,
+            String bucketName
+    ) {
+        s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secret));
+        this.bucketName = bucketName;
+    }
+
+    @Override
+    public void upload(
+            String projectName,
+            InputStream uploadStream,
+            long contentLength
+    ) {
+        ObjectMetadata metadata = new ObjectMetadata();
+        metadata.setContentLength(contentLength);
+        PutObjectRequest put = new PutObjectRequest(
+                bucketName,
+                projectName,
+                uploadStream,
+                metadata
+        );
+        PutObjectResult res = s3.putObject(put);
+    }
+
+    @Override
+    public InputStream openDownloadStream(String projectName) {
+        GetObjectRequest get = new GetObjectRequest(
+                bucketName,
+                projectName
+        );
+        S3Object res = s3.getObject(get);
+        return res.getObjectContent();
+    }
+
+    @Override
+    public void remove(String projectName) {
+        DeleteObjectRequest del = new DeleteObjectRequest(
+                bucketName,
+                projectName
+        );
+        s3.deleteObject(del);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java

@@ -1,11 +1,13 @@
 package uk.ac.ic.wlgitbridge.bridge.swap;
 
+import java.time.Duration;
+
 /**
  * Created by winston on 20/08/2016.
  */
 public interface SwapJob {
 
-    void start(int intervalMillis);
+    void start(Duration interval);
 
     void stop();
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java

@@ -3,9 +3,10 @@ package uk.ac.ic.wlgitbridge.bridge.swap;
 import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
 import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
-import uk.ac.ic.wlgitbridge.util.Util;
 
+import java.time.Duration;
 import java.util.Timer;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Created by winston on 20/08/2016.
@@ -19,10 +20,13 @@ public class SwapJobImpl implements SwapJob {
 
     private final Timer timer;
 
+    final AtomicInteger swaps;
+
     public SwapJobImpl(
             ProjectLock lock,
             RepoStore repoStore,
-            DBStore dbStore, SwapStore swapStore
+            DBStore dbStore,
+            SwapStore swapStore
     ) {
 
         this.lock = lock;
@@ -30,14 +34,15 @@ public class SwapJobImpl implements SwapJob {
         this.swapStore = swapStore;
         this.dbStore = dbStore;
         timer = new Timer();
+        swaps = new AtomicInteger(0);
     }
 
     @Override
-    public void start(int intervalMillis) {
+    public void start(Duration interval) {
         timer.scheduleAtFixedRate(
-                Util.makeTimerTask(this::doSwap),
+                uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
                 0,
-                intervalMillis
+                interval.toMillis()
         );
     }
 
@@ -47,7 +52,7 @@ public class SwapJobImpl implements SwapJob {
     }
 
     private void doSwap() {
-        throw new UnsupportedOperationException();
+        swaps.incrementAndGet();
     }
 
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java

@@ -1,7 +1,16 @@
 package uk.ac.ic.wlgitbridge.bridge.swap;
 
+import java.io.InputStream;
+
 /**
  * Created by winston on 20/08/2016.
  */
 public interface SwapStore {
+
+    void upload(String projectName, InputStream uploadStream, long contentLength);
+
+    InputStream openDownloadStream(String projectName);
+
+    void remove(String projectName);
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java

@@ -24,6 +24,7 @@ import javax.servlet.DispatcherType;
 import javax.servlet.Filter;
 import javax.servlet.ServletException;
 import java.io.File;
+import java.io.InputStream;
 import java.net.BindException;
 import java.util.EnumSet;
 
@@ -50,7 +51,22 @@ public class GitBridgeServer {
         this.rootGitDirectoryPath = config.getRootGitDirectory();
         RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
         DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
-        SwapStore swapStore = new SwapStore() {};
+        SwapStore swapStore = new SwapStore() {
+            @Override
+            public void upload(String projectName, InputStream uploadStream, long contentLength) {
+
+            }
+
+            @Override
+            public InputStream openDownloadStream(String projectName) {
+                return null;
+            }
+
+            @Override
+            public void remove(String projectName) {
+
+            }
+        };
         bridgeAPI = Bridge.make(
                 repoStore,
                 dbStore,

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java

@@ -0,0 +1,127 @@
+package uk.ac.ic.wlgitbridge.util;
+
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class Files {
+
+    private Files() {}
+
+    public static boolean contentsAreEqual(
+            File f0,
+            File f1
+    ) throws IOException {
+        try {
+            return uncheckedContentsAreEqual(f0, f1);
+        } catch (UncheckedIOException e) {
+            throw e.getCause();
+        }
+    }
+
+    public static void renameAll(
+            File fileOrDir,
+            String from,
+            String to
+    ) {
+        if (fileOrDir.isDirectory()) {
+            File f = doRename(fileOrDir, from, to);
+            for (File c : f.listFiles()) {
+                renameAll(c, from, to);
+            }
+        } else if (fileOrDir.isFile()) {
+            doRename(fileOrDir, from, to);
+        } else {
+            throw new IllegalArgumentException(
+                    "not a file or dir: " + fileOrDir
+            );
+        }
+    }
+
+    private static File doRename(File fileOrDir, String from, String to) {
+        if (!fileOrDir.getName().equals(from)) {
+            return fileOrDir;
+        }
+        File renamed = new File(fileOrDir.getParent(), to);
+        Preconditions.checkState(fileOrDir.renameTo(renamed));
+        return renamed;
+    }
+
+    private static boolean uncheckedContentsAreEqual(File f0, File f1) throws IOException {
+        if (f0.equals(f1)) {
+            return true;
+        }
+        if (!f0.isDirectory() || !f1.isDirectory()) {
+            return !f0.isDirectory() && !f1.isDirectory() &&
+                    Arrays.equals(
+                            FileUtils.readFileToByteArray(f0),
+                            FileUtils.readFileToByteArray(f1)
+                    );
+        }
+        Path f0Base = Paths.get(f0.getAbsolutePath());
+        Path f1Base = Paths.get(f1.getAbsolutePath());
+        Set<Path> children0 = getChildren(f0, f0Base);
+        Set<Path> children1 = getChildren(f1, f1Base);
+        if (children0.size() != children1.size()) {
+            return false;
+        }
+        return children0.stream(
+        ).allMatch(c0 ->
+                children1.contains(c0) && childEquals(c0, f0Base, f1Base)
+        );
+    }
+
+    private static Set<Path> getChildren(File f0, Path f0Base) {
+        return FileUtils.listFilesAndDirs(
+                f0,
+                TrueFileFilter.TRUE,
+                TrueFileFilter.TRUE
+        ).stream(
+        ).map(
+                File::getAbsolutePath
+        ).map(
+                Paths::get
+        ).map(p ->
+                f0Base.relativize(p)
+        ).filter(p ->
+                !p.toString().isEmpty()
+        ).collect(
+                Collectors.toSet()
+        );
+    }
+
+    private static boolean childEquals(
+            Path child,
+            Path f0Base,
+            Path f1Base
+    ) throws UncheckedIOException {
+        File c0 = f0Base.resolve(child).toFile();
+        File c1 = f1Base.resolve(child).toFile();
+        boolean c0IsDir = c0.isDirectory();
+        boolean c1IsDir = c1.isDirectory();
+        if (c0IsDir || c1IsDir) {
+            return c0IsDir && c1IsDir;
+        }
+        try {
+            return c0.isFile() && c1.isFile() && Arrays.equals(
+                    FileUtils.readFileToByteArray(c0),
+                    FileUtils.readFileToByteArray(c1)
+            );
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java

@@ -0,0 +1,13 @@
+package uk.ac.ic.wlgitbridge.util;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class Project {
+
+    public static boolean isValidProjectName(String projectName) {
+        return projectName != null && !projectName.isEmpty()
+                && !projectName.startsWith(".");
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java

@@ -0,0 +1,106 @@
+package uk.ac.ic.wlgitbridge.util;
+
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
+
+import java.io.*;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class Tar {
+
+    private Tar() {}
+
+    public static InputStream tar(File fileOrDir) throws IOException {
+        ByteArrayOutputStream bout = new ByteArrayOutputStream();
+        TarArchiveOutputStream tout = new TarArchiveOutputStream(bout);
+        addTarEntry(
+                tout,
+                Paths.get(fileOrDir.getParentFile().getAbsolutePath()),
+                fileOrDir
+        );
+        tout.close();
+        return new ByteArrayInputStream(bout.toByteArray());
+    }
+
+    private static void addTarEntry(
+            TarArchiveOutputStream tout,
+            Path base,
+            File fileOrDir
+    ) throws IOException {
+        if (fileOrDir.isDirectory()) {
+            addTarDir(tout, base, fileOrDir);
+        } else if (fileOrDir.isFile()) {
+            addTarFile(tout, base, fileOrDir);
+        } else {
+            throw new IllegalArgumentException(
+                    "invalid file or dir: " + fileOrDir
+            );
+        }
+    }
+
+    private static void addTarDir(
+            TarArchiveOutputStream tout,
+            Path base,
+            File dir
+    ) throws IOException {
+        Preconditions.checkArgument(dir.isDirectory());
+        String name = base.relativize(
+                Paths.get(dir.getAbsolutePath())
+        ).toString();
+        ArchiveEntry entry = tout.createArchiveEntry(dir, name);
+        tout.putArchiveEntry(entry);
+        tout.closeArchiveEntry();
+        for (File f : dir.listFiles()) {
+            addTarEntry(tout, base, f);
+        }
+    }
+
+    private static void addTarFile(
+            TarArchiveOutputStream tout,
+            Path base,
+            File file
+    ) throws IOException {
+        Preconditions.checkArgument(file.isFile());
+        String name = base.relativize(
+                Paths.get(file.getAbsolutePath())
+        ).toString();
+        ArchiveEntry entry = tout.createArchiveEntry(file, name);
+        tout.putArchiveEntry(entry);
+        tout.write(FileUtils.readFileToByteArray(file));
+        tout.closeArchiveEntry();
+    }
+
+    public static void untar(InputStream tar, File parentDir) throws IOException {
+        TarArchiveInputStream tin = new TarArchiveInputStream(tar);
+        ArchiveEntry e;
+        while ((e = tin.getNextEntry()) != null) {
+            File f = new File(parentDir, e.getName());
+            f.setLastModified(e.getLastModifiedDate().getTime());
+            f.getParentFile().mkdirs();
+            if (e.isDirectory()) {
+                f.mkdir();
+                continue;
+            }
+            long size = e.getSize();
+            Preconditions.checkArgument(
+                    size > 0 && size < Integer.MAX_VALUE,
+                    "file too big: tar should have thrown an IOException"
+            );
+            try (OutputStream out = new FileOutputStream(f)) {
+                /* TarInputStream pretends each
+                   entry's EOF is the stream's EOF */
+                IOUtils.copy(tin, out);
+            }
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java

@@ -0,0 +1,19 @@
+package uk.ac.ic.wlgitbridge.util;
+
+import java.util.TimerTask;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class Timer {
+
+    public static TimerTask makeTimerTask(Runnable lamb) {
+        return new TimerTask() {
+            @Override
+            public void run() {
+                lamb.run();
+            }
+        };
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java

@@ -19,15 +19,6 @@ public class Util {
     private static String POSTBACK_URL;
     private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
 
-    public static TimerTask makeTimerTask(Runnable lamb) {
-        return new TimerTask() {
-            @Override
-            public void run() {
-                lamb.run();
-            }
-        };
-    }
-
     public static String entries(int entries) {
         if (entries == 1) {
             return "entry";

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java

@@ -10,6 +10,8 @@ import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
 import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
 import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
 
+import java.time.Duration;
+
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -35,6 +37,7 @@ public class BridgeTest {
         dbStore = mock(DBStore.class);
         swapStore = mock(SwapStore.class);
         snapshotAPI = mock(SnapshotAPI.class);
+        resourceCache = mock(ResourceCache.class);
         swapJob = mock(SwapJob.class);
         bridge = new Bridge(
                 lock,
@@ -49,9 +52,9 @@ public class BridgeTest {
 
     @Test
     public void shutdownStopsSwapJob() {
-        bridge.startSwapJob(1000);
+        bridge.startSwapJob(Duration.ofSeconds(1));
         bridge.doShutdown();
-        verify(swapJob).start(1000);
+        verify(swapJob).start(Duration.ofSeconds(1));
         verify(swapJob).stop();
     }
 

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java

@@ -0,0 +1,75 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import uk.ac.ic.wlgitbridge.util.Files;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class FSRepoStoreTest {
+
+    private FSRepoStore repoStore;
+    private File original;
+
+    @Before
+    public void setup() throws IOException {
+        TemporaryFolder tmpFolder = new TemporaryFolder();
+        tmpFolder.create();
+        File tmp = tmpFolder.newFolder("repostore");
+        Path rootdir = Paths.get(
+                "src/test/resources/uk/ac/ic/wlgitbridge/"
+                        + "bridge/repo/FSRepoStoreTest/rootdir"
+        );
+        FileUtils.copyDirectory(rootdir.toFile(), tmp);
+        Files.renameAll(tmp, "DOTgit", ".git");
+        original = tmpFolder.newFolder("original");
+        FileUtils.copyDirectory(tmp, original);
+        repoStore = new FSRepoStore(tmp.getAbsolutePath());
+    }
+
+    @Test
+    public void testPurgeNonexistentProjects() {
+        File toDelete = new File(repoStore.getRootDirectory(), "idontexist");
+        File wlgb = new File(repoStore.getRootDirectory(), ".wlgb");
+        Assert.assertTrue(toDelete.exists());
+        Assert.assertTrue(wlgb.exists());
+        repoStore.purgeNonexistentProjects(Arrays.asList("proj1", "proj2"));
+        Assert.assertFalse(toDelete.exists());
+        Assert.assertTrue(wlgb.exists());
+    }
+
+    @Test
+    public void testTotalSize() {
+        assertEquals(31860, repoStore.totalSize());
+    }
+
+    @Test
+    public void zipAndUnzipShouldBeTheSame() throws IOException {
+        long beforeSize = repoStore.totalSize();
+        InputStream zipped = repoStore.bzip2Project("proj1");
+        repoStore.remove("proj1");
+        Assert.assertTrue(beforeSize > repoStore.totalSize());
+        repoStore.unbzip2Project("proj1", zipped);
+        Assert.assertEquals(beforeSize, repoStore.totalSize());
+        Assert.assertTrue(
+                Files.contentsAreEqual(
+                        original,
+                        repoStore.getRootDirectory()
+                )
+        );
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java

@@ -0,0 +1,41 @@
+package uk.ac.ic.wlgitbridge.bridge.swap;
+
+import org.junit.Before;
+
+/**
+ * Created by winston on 21/08/2016.
+ */
+public class S3SwapStoreTest {
+
+    private static final String accessKey = null;
+    private static final String secret = null;
+    private static final String bucketName = "com.overleaf.testbucket";
+
+    private S3SwapStore s3;
+
+    @Before
+    public void setup() {
+        if (accessKey == null || secret == null) {
+            s3 = null;
+            return;
+        }
+        s3 = new S3SwapStore(accessKey, secret, bucketName);
+    }
+
+//    @Ignore
+//    @Test
+//    public void testUploadDownloadDelete() throws Exception {
+//        assumeNotNull(s3);
+//        String projName = "abc123";
+//        byte[] contents = "hello".getBytes();
+//        s3.upload(
+//                projName,
+//                new ByteArrayInputStream(contents),
+//                contents.length
+//        );
+//        InputStream down = s3.openDownloadStream(projName);
+//        s3.remove(projName);
+//        assertArrayEquals(contents, IOUtils.toByteArray(down));
+//    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java

@@ -0,0 +1,57 @@
+package uk.ac.ic.wlgitbridge.bridge.swap;
+
+import org.junit.Before;
+import org.junit.Test;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class SwapJobImplTest {
+
+    private SwapJobImpl swapJob;
+
+    private ProjectLock lock;
+    private RepoStore repoStore;
+    private DBStore dbStore;
+    private SwapStore swapStore;
+
+    @Before
+    public void setup() {
+        lock = mock(ProjectLock.class);
+        repoStore = mock(RepoStore.class);
+        dbStore = mock(DBStore.class);
+        swapStore = mock(SwapStore.class);
+        swapJob = new SwapJobImpl(
+                lock,
+                repoStore,
+                dbStore,
+                swapStore
+        );
+    }
+
+    @Test
+    public void startingTimerAlwaysCausesASwap() {
+        assertEquals(0, swapJob.swaps.get());
+        swapJob.start(Duration.ofHours(1));
+        while (swapJob.swaps.get() <= 0);
+        assertTrue(swapJob.swaps.get() > 0);
+    }
+
+    @Test
+    public void swapsHappenEveryInterval() {
+        assertEquals(0, swapJob.swaps.get());
+        swapJob.start(Duration.ofMillis(1));
+        while (swapJob.swaps.get() <= 1);
+        assertTrue(swapJob.swaps.get() > 1);
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java

@@ -0,0 +1,18 @@
+package uk.ac.ic.wlgitbridge.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class ProjectTest {
+
+    @Test
+    public void testValidProjectNames() {
+        Assert.assertFalse(Project.isValidProjectName(null));
+        Assert.assertFalse(Project.isValidProjectName(""));
+        Assert.assertFalse(Project.isValidProjectName(".wlgb"));
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java

@@ -0,0 +1,45 @@
+package uk.ac.ic.wlgitbridge.util;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class TarTest {
+
+    private File testDir;
+
+    @Before
+    public void setup() throws IOException {
+        TemporaryFolder tmpFolder = new TemporaryFolder();
+        tmpFolder.create();
+        testDir = tmpFolder.newFolder("testdir");
+        Path resdir = Paths.get(
+                "src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir"
+        );
+        FileUtils.copyDirectory(resdir.toFile(), testDir);
+    }
+
+    @Test
+    public void tarAndUntarProducesTheSameResult() throws IOException {
+        InputStream tar = Tar.tar(testDir);
+        TemporaryFolder tmpF = new TemporaryFolder();
+        tmpF.create();
+        File parentDir = tmpF.newFolder();
+        Tar.untar(tar, parentDir);
+        File untarred = new File(parentDir, "testdir");
+        assertTrue(Files.contentsAreEqual(testDir, untarred));
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java

@@ -0,0 +1,19 @@
+package uk.ac.ic.wlgitbridge.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class TimerTest {
+
+    @Test
+    public void testMakeTimerTask() {
+        int[] iPtr = new int[] { 3 };
+        Timer.makeTimerTask(() -> iPtr[0] = 5).run();
+        assertEquals(5, iPtr[0]);
+    }
+
+}

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj1

@@ -0,0 +1 @@
+Subproject commit e5fc0d2678ec7b9bacf0bf514bac035fa371cb6e

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj2

@@ -0,0 +1 @@
+Subproject commit 6c12c073e5702530a9d06b83840d62f8a6621764

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1

@@ -0,0 +1 @@
+file1

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2

@@ -0,0 +1 @@
+file2

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1

@@ -0,0 +1 @@
+nest1/file1

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2

@@ -0,0 +1 @@
+nest1file2

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3

@@ -0,0 +1 @@
+nest1/file3

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1

@@ -0,0 +1 @@
+nest1/nest2/file1

services/git-bridge/writelatex-git-bridge.iml

@@ -37,10 +37,6 @@
     <orderEntry type="library" name="Maven: org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r" level="project" />
     <orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.53" level="project" />
     <orderEntry type="library" name="Maven: com.googlecode.javaewah:JavaEWAH:0.7.9" level="project" />
-    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.3.6" level="project" />
-    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.3.3" level="project" />
-    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
-    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.6" level="project" />
     <orderEntry type="library" name="Maven: org.eclipse.jgit:org.eclipse.jgit.http.server:4.4.1.201607150455-r" level="project" />
     <orderEntry type="library" name="Maven: org.xerial:sqlite-jdbc:3.8.11.2" level="project" />
     <orderEntry type="library" name="Maven: joda-time:joda-time:2.9.4" level="project" />
@@ -80,9 +76,9 @@
     <orderEntry type="library" scope="TEST" name="Maven: com.jcraft:jzlib:1.1.3" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcmail-jdk15on:1.52" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcpkix-jdk15on:1.52" level="project" />
-    <orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.3" level="project" />
-    <orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.3" level="project" />
-    <orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.3" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.3" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.3" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.3" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.2.3" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.json:json:20090211" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: com.github.fge:json-schema-validator:2.2.6" level="project" />
@@ -97,7 +93,6 @@
     <orderEntry type="library" scope="TEST" name="Maven: javax.activation:activation:1.1" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: net.sf.jopt-simple:jopt-simple:4.6" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: xerces:xercesImpl:2.4.0" level="project" />
-    <orderEntry type="library" scope="TEST" name="Maven: commons-io:commons-io:1.3.2" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-buffer:4.0.34.Final" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-codec:4.0.34.Final" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-codec-http:4.0.34.Final" level="project" />
@@ -109,5 +104,84 @@
     <orderEntry type="library" scope="TEST" name="Maven: org.mock-server:mockserver-logging:3.10.4" level="project" />
     <orderEntry type="library" name="Maven: org.mockito:mockito-core:1.10.19" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="Maven: org.objenesis:objenesis:2.1" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-support:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:jmespath-java:1.0" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-simpledb:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-servicecatalog:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-simpleworkflow:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-storagegateway:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-route53:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-s3:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-importexport:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-sts:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-sqs:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-rds:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-redshift:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticbeanstalk:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-glacier:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-iam:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-datapipeline:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticloadbalancing:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticloadbalancingv2:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-emr:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticache:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elastictranscoder:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ec2:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-dynamodb:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-sns:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudtrail:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudwatch:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-logs:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-events:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cognitoidentity:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cognitosync:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-directconnect:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudformation:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudfront:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-kinesis:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-opsworks:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ses:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-autoscaling:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudsearch:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudwatchmetrics:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-codedeploy:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-codepipeline:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-kms:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-config:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-lambda:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ecs:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ecr:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudhsm:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ssm:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-workspaces:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-machinelearning:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-directory:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-efs:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-codecommit:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-devicefarm:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticsearch:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-waf:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-marketplacecommerceanalytics:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-inspector:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-iot:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-api-gateway:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-acm:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-gamelift:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-dms:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-marketplacemeteringservice:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cognitoidp:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-discovery:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-applicationautoscaling:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-snowball:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-core:1.11.28" level="project" />
+    <orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.6" level="project" />
+    <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-swf-libraries:1.11.22" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.2" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.4" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
+    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.9" level="project" />
+    <orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.12" level="project" />
   </component>
 </module>