Merge pull request #19 from overleaf/decouple-refactor

Decouple and Refactor

Separating a couple of commits from #17 to make it more manageable.

- Decouple jgit's Repository class from the Bridge
- Large refactor of parts into distinct components / interfaces

services/git-bridge/pom.xml

@@ -134,5 +134,11 @@
             <version>3.10.4</version>
             <scope>test</scope>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>1.10.19</version>
+        </dependency>
     </dependencies>
 </project>

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java

@@ -0,0 +1,363 @@
+package uk.ac.ic.wlgitbridge.bridge;
+
+import com.google.api.client.auth.oauth2.Credential;
+import org.eclipse.jgit.transport.ServiceMayNotContinueException;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
+import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
+import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
+import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl;
+import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
+import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
+import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.model.Snapshot;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
+import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Created by Winston on 16/11/14.
+ */
+public class Bridge {
+
+    private final ProjectLock lock;
+
+    private final RepoStore repoStore;
+    private final DBStore dbStore;
+    private final SwapStore swapStore;
+
+    private final SnapshotAPI snapshotAPI;
+    private final ResourceCache resourceCache;
+
+    private final SwapJob swapJob;
+
+    private final PostbackManager postbackManager;
+
+    public static Bridge make(
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore
+    ) {
+        ProjectLock lock = new ProjectLockImpl((int threads) ->
+                Log.info("Waiting for " + threads + " projects...")
+        );
+        return new Bridge(
+                lock,
+                repoStore,
+                dbStore,
+                swapStore,
+                new NetSnapshotAPI(),
+                new UrlResourceCache(dbStore),
+                new SwapJobImpl(
+                        lock,
+                        repoStore,
+                        dbStore,
+                        swapStore
+                )
+        );
+    }
+
+    Bridge(
+            ProjectLock lock,
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore,
+            SnapshotAPI snapshotAPI,
+            ResourceCache resourceCache,
+            SwapJob swapJob
+    ) {
+        this.lock = lock;
+        this.repoStore = repoStore;
+        this.dbStore = dbStore;
+        this.swapStore = swapStore;
+        this.snapshotAPI = snapshotAPI;
+        this.resourceCache = resourceCache;
+        this.swapJob = swapJob;
+        postbackManager = new PostbackManager();
+        Runtime.getRuntime().addShutdownHook(new Thread(this::doShutdown));
+        repoStore.purgeNonexistentProjects(dbStore.getProjectNames());
+    }
+
+    void doShutdown() {
+        Log.info("Shutdown received.");
+        Log.info("Stopping SwapJob");
+        swapJob.stop();
+        Log.info("Waiting for projects");
+        lock.lockAll();
+        Log.info("Bye");
+    }
+
+    public void startSwapJob(int intervalMillis) {
+        swapJob.start(intervalMillis);
+    }
+
+    /* TODO: Remove these when WLBridged is moved into RepoStore */
+    public void lockForProject(String projectName) {
+        lock.lockForProject(projectName);
+    }
+
+    public void unlockForProject(String projectName) {
+        lock.unlockForProject(projectName);
+    }
+
+    public boolean repositoryExists(Credential oauth2, String projectName)
+            throws ServiceMayNotContinueException, GitUserException {
+        lockForProject(projectName);
+        GetDocRequest getDocRequest = new GetDocRequest(oauth2, projectName);
+        getDocRequest.request();
+        try {
+            getDocRequest.getResult().getVersionID();
+        } catch (InvalidProjectException e) {
+            return false;
+        } finally {
+            unlockForProject(projectName);
+        }
+        return true;
+    }
+
+    public void getWritableRepositories(
+            Credential oauth2,
+            ProjectRepo repo
+    ) throws IOException,
+             GitUserException {
+        Log.info("[{}] Fetching", repo.getProjectName());
+        updateProjectWithName(oauth2, repo);
+    }
+
+    public void
+    putDirectoryContentsToProjectWithName(Credential oauth2,
+                                          String projectName,
+                                          RawDirectory directoryContents,
+                                          RawDirectory oldDirectoryContents,
+                                          String hostname)
+            throws SnapshotPostException, IOException, ForbiddenException {
+        lock.lockForProject(projectName);
+        CandidateSnapshot candidate = null;
+        try {
+            Log.info("[{}] Pushing", projectName);
+            String postbackKey = postbackManager.makeKeyForProject(projectName);
+            Log.info(
+                    "[{}] Created postback key: {}",
+                    projectName,
+                    postbackKey
+            );
+            candidate =
+                    createCandidateSnapshot(
+                            projectName,
+                            directoryContents,
+                            oldDirectoryContents
+                    );
+            Log.info(
+                    "[{}] Candindate snapshot created: {}",
+                    projectName,
+                    candidate
+            );
+            PushRequest pushRequest = new PushRequest(
+                    oauth2,
+                    candidate,
+                    postbackKey
+            );
+            pushRequest.request();
+            PushResult result = pushRequest.getResult();
+            if (result.wasSuccessful()) {
+                Log.info(
+                        "[{}] Push to Overleaf successful",
+                        projectName
+                );
+                Log.info("[{}] Waiting for postback...", projectName);
+                int versionID =
+                        postbackManager.waitForVersionIdOrThrow(projectName);
+                Log.info(
+                        "[{}] Got version ID for push: {}",
+                        projectName,
+                        versionID
+                );
+                approveSnapshot(versionID, candidate);
+                Log.info(
+                        "[{}] Approved version ID: {}",
+                        projectName,
+                        versionID
+                );
+            } else {
+                Log.warn(
+                        "[{}] Went out of date while waiting for push",
+                        projectName
+                );
+                throw new OutOfDateException();
+            }
+        } catch (SevereSnapshotPostException e) {
+            Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
+            throw e;
+        } catch (SnapshotPostException e) {
+            /* Stack trace should be printed further up */
+            Log.warn(
+                    "[{}] Exception when waiting for postback: {}",
+                    projectName,
+                    e.getClass().getSimpleName()
+            );
+            throw e;
+        } catch (IOException e) {
+            Log.warn("[{}] IOException on put", projectName);
+            throw e;
+        } finally {
+            if (candidate != null) {
+                candidate.deleteServletFiles();
+            } else {
+                Log.error(
+                        "[{}] Candidate snapshot was null: " +
+                                "this should never happen.",
+                        projectName
+                );
+            }
+            lock.unlockForProject(projectName);
+        }
+    }
+
+    public void checkPostbackKey(String projectName, String postbackKey)
+            throws InvalidPostbackKeyException {
+        postbackManager.checkPostbackKey(projectName, postbackKey);
+    }
+
+    /* Called by postback thread. */
+    public void postbackReceivedSuccessfully(String projectName,
+                                             String postbackKey,
+                                             int versionID)
+            throws UnexpectedPostbackException {
+        Log.info(
+                "[{}]" +
+                        " Postback received by postback thread, version: {}",
+                projectName,
+                versionID);
+        postbackManager.postVersionIDForProject(
+                projectName,
+                versionID,
+                postbackKey
+        );
+    }
+
+    public void postbackReceivedWithException(String projectName,
+                                              String postbackKey,
+                                              SnapshotPostException exception)
+            throws UnexpectedPostbackException {
+        Log.warn("[{}] Postback received with exception", projectName);
+        postbackManager.postExceptionForProject(
+                projectName,
+                exception,
+                postbackKey
+        );
+    }
+
+    /* PRIVATE */
+
+    private void updateProjectWithName(
+            Credential oauth2,
+            ProjectRepo repo
+    ) throws IOException, GitUserException {
+        String projectName = repo.getProjectName();
+        Deque<Snapshot> snapshots =
+                snapshotAPI.getSnapshotsForProjectAfterVersion(
+                        oauth2,
+                        projectName,
+                        dbStore.getLatestVersionForProject(projectName)
+                );
+
+        makeCommitsFromSnapshots(repo, snapshots);
+
+        if (!snapshots.isEmpty()) {
+            dbStore.setLatestVersionForProject(
+                    projectName,
+                    snapshots.getLast().getVersionID()
+            );
+        }
+    }
+
+    private void makeCommitsFromSnapshots(ProjectRepo repo,
+                                          Collection<Snapshot> snapshots)
+            throws IOException, GitUserException {
+        String name = repo.getProjectName();
+        for (Snapshot snapshot : snapshots) {
+            Map<String, RawFile> fileTable = repo.getFiles();
+            List<RawFile> files = new LinkedList<>();
+            files.addAll(snapshot.getSrcs());
+            Map<String, byte[]> fetchedUrls = new HashMap<>();
+            for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
+                files.add(
+                        resourceCache.get(
+                                name,
+                                snapshotAttachment.getUrl(),
+                                snapshotAttachment.getPath(),
+                                fileTable,
+                                fetchedUrls
+                        )
+                );
+            }
+            Log.info(
+                    "[{}] Committing version ID: {}",
+                    name,
+                    snapshot.getVersionID()
+            );
+            Collection<String> missingFiles = repo.commitAndGetMissing(
+                    new GitDirectoryContents(
+                            files,
+                            repoStore.getRootDirectory(),
+                            name,
+                            snapshot
+                    )
+            );
+            dbStore.deleteFilesForProject(
+                    name,
+                    missingFiles.toArray(new String[missingFiles.size()])
+            );
+        }
+    }
+
+    private CandidateSnapshot createCandidateSnapshot(
+            String projectName,
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents
+    ) throws IOException {
+        CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
+                projectName,
+                dbStore.getLatestVersionForProject(projectName),
+                directoryContents,
+                oldDirectoryContents
+        );
+        candidateSnapshot.writeServletFiles(repoStore.getRootDirectory());
+        return candidateSnapshot;
+    }
+
+    private void approveSnapshot(
+            int versionID,
+            CandidateSnapshot candidateSnapshot
+    ) {
+        List<String> deleted = candidateSnapshot.getDeleted();
+        dbStore.setLatestVersionForProject(
+                candidateSnapshot.getProjectName(),
+                versionID
+        );
+        dbStore.deleteFilesForProject(
+                candidateSnapshot.getProjectName(),
+                deleted.toArray(new String[deleted.size()])
+        );
+    }
+
+
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/BridgeAPI.java

@@ -1,201 +0,0 @@
-package uk.ac.ic.wlgitbridge.bridge;
-
-import com.google.api.client.auth.oauth2.Credential;
-import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.transport.ServiceMayNotContinueException;
-import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
-import uk.ac.ic.wlgitbridge.data.ProjectLock;
-import uk.ac.ic.wlgitbridge.data.ShutdownHook;
-import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
-import uk.ac.ic.wlgitbridge.data.model.DataStore;
-import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
-import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
-import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
-import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
-import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
-import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
-import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
-import uk.ac.ic.wlgitbridge.util.Log;
-
-import java.io.IOException;
-
-/**
- * Created by Winston on 16/11/14.
- */
-public class BridgeAPI {
-
-    private final DataStore dataStore;
-    private final PostbackManager postbackManager;
-    private final ProjectLock mainProjectLock;
-
-    public BridgeAPI(String rootGitDirectoryPath) {
-        dataStore = new DataStore(rootGitDirectoryPath);
-        postbackManager = new PostbackManager();
-        mainProjectLock = new ProjectLock();
-        Runtime.getRuntime().addShutdownHook(new ShutdownHook(mainProjectLock));
-    }
-
-    public void lockForProject(String projectName) {
-        mainProjectLock.lockForProject(projectName);
-    }
-
-    public void unlockForProject(String projectName) {
-        mainProjectLock.unlockForProject(projectName);
-    }
-
-    public boolean repositoryExists(Credential oauth2, String projectName)
-            throws ServiceMayNotContinueException, ForbiddenException {
-        lockForProject(projectName);
-        GetDocRequest getDocRequest = new GetDocRequest(oauth2, projectName);
-        getDocRequest.request();
-        try {
-            getDocRequest.getResult().getVersionID();
-        } catch (InvalidProjectException e) {
-            return false;
-        } catch (FailedConnectionException e) {
-            throw e;
-        } catch (SnapshotPostException e) {
-            throw new ServiceMayNotContinueException(e.getMessage());
-        } finally {
-            unlockForProject(projectName);
-        }
-        return true;
-    }
-
-    public void getWritableRepositories(Credential oauth2,
-                                        String projectName,
-                                        Repository repository)
-            throws IOException,
-                   SnapshotPostException,
-                   GitAPIException,
-                   ForbiddenException {
-        Log.info("[{}] Fetching", projectName);
-        dataStore.updateProjectWithName(oauth2, projectName, repository);
-    }
-
-    public void
-    putDirectoryContentsToProjectWithName(Credential oauth2,
-                                          String projectName,
-                                          RawDirectory directoryContents,
-                                          RawDirectory oldDirectoryContents,
-                                          String hostname)
-            throws SnapshotPostException, IOException, ForbiddenException {
-        mainProjectLock.lockForProject(projectName);
-        CandidateSnapshot candidate = null;
-        try {
-            Log.info("[{}] Pushing", projectName);
-            String postbackKey = postbackManager.makeKeyForProject(projectName);
-            Log.info(
-                    "[{}] Created postback key: {}",
-                    projectName,
-                    postbackKey
-            );
-            candidate =
-                    dataStore.createCandidateSnapshot(
-                            projectName,
-                            directoryContents,
-                            oldDirectoryContents
-                    );
-            Log.info(
-                    "[{}] Candindate snapshot created: {}",
-                    projectName,
-                    candidate
-            );
-            PushRequest pushRequest = new PushRequest(
-                    oauth2,
-                    candidate,
-                    postbackKey
-            );
-            pushRequest.request();
-            PushResult result = pushRequest.getResult();
-            if (result.wasSuccessful()) {
-                Log.info(
-                        "[{}] Push to Overleaf successful",
-                        projectName
-                );
-                Log.info("[{}] Waiting for postback...", projectName);
-                int versionID =
-                        postbackManager.waitForVersionIdOrThrow(projectName);
-                Log.info(
-                        "[{}] Got version ID for push: {}",
-                        projectName,
-                        versionID
-                );
-                dataStore.approveSnapshot(versionID, candidate);
-                Log.info(
-                        "[{}] Approved version ID: {}",
-                        projectName,
-                        versionID
-                );
-            } else {
-                Log.warn(
-                        "[{}] Went out of date while waiting for push",
-                        projectName
-                );
-                throw new OutOfDateException();
-            }
-        } catch (SevereSnapshotPostException e) {
-            Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
-            throw e;
-        } catch (SnapshotPostException e) {
-            /* Stack trace should be printed further up */
-            Log.warn(
-                    "[{}] Exception when waiting for postback: {}",
-                    projectName,
-                    e.getClass().getSimpleName()
-            );
-            throw e;
-        } catch (IOException e) {
-            Log.warn("[{}] IOException on put", projectName);
-            throw e;
-        } finally {
-            if (candidate != null) {
-                candidate.deleteServletFiles();
-            } else {
-                Log.error(
-                        "[{}] Candidate snapshot was null: " +
-                                "this should never happen.",
-                        projectName
-                );
-            }
-            mainProjectLock.unlockForProject(projectName);
-        }
-    }
-
-    public void checkPostbackKey(String projectName, String postbackKey)
-            throws InvalidPostbackKeyException {
-        postbackManager.checkPostbackKey(projectName, postbackKey);
-    }
-
-    /* Called by postback thread. */
-    public void postbackReceivedSuccessfully(String projectName,
-                                             String postbackKey,
-                                             int versionID)
-            throws UnexpectedPostbackException {
-        Log.info(
-                "[{}]" +
-                        " Postback received by postback thread, version: {}",
-                projectName,
-                versionID);
-        postbackManager.postVersionIDForProject(
-                projectName,
-                versionID,
-                postbackKey
-        );
-    }
-
-    public void postbackReceivedWithException(String projectName,
-                                              String postbackKey,
-                                              SnapshotPostException exception)
-            throws UnexpectedPostbackException {
-        Log.warn("[{}] Postback received with exception", projectName);
-        postbackManager.postExceptionForProject(
-                projectName,
-                exception,
-                postbackKey
-        );
-    }
-
-}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/GitProjectRepo.java

@@ -0,0 +1,93 @@
+package uk.ac.ic.wlgitbridge.bridge;
+
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.PersonIdent;
+import org.eclipse.jgit.lib.Repository;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class GitProjectRepo implements ProjectRepo {
+
+    private final Repository repository;
+    private final String projectName;
+
+    public GitProjectRepo(Repository repository, String projectName) {
+        this.repository = repository;
+        this.projectName = projectName;
+    }
+
+    @Override
+    public String getProjectName() {
+        return projectName;
+    }
+
+    @Override
+    public Map<String, RawFile> getFiles()
+            throws IOException, GitUserException {
+        return new RepositoryObjectTreeWalker(
+                repository
+        ).getDirectoryContents().getFileTable();
+    }
+
+    @Override
+    public Collection<String> commitAndGetMissing(
+            GitDirectoryContents contents
+    ) throws IOException {
+        try {
+            return doCommitAndGetMissing(contents);
+        } catch (GitAPIException e) {
+            throw new IOException(e);
+        }
+    }
+
+    private Collection<String> doCommitAndGetMissing(
+            GitDirectoryContents contents
+    ) throws IOException, GitAPIException {
+        String name = getProjectName();
+        Log.info("[{}] Writing commit", name);
+        contents.write();
+        Git git = new Git(repository);
+        Log.info("[{}] Getting missing files", name);
+        Set<String> missingFiles = git.status().call().getMissing();
+        for (String missing : missingFiles) {
+            Log.info("[{}] Git rm {}", name, missing);
+            git.rm().setCached(true).addFilepattern(missing).call();
+        }
+        Log.info("[{}] Calling Git add", name);
+        git.add().addFilepattern(".").call();
+        Log.info("[{}] Calling Git commit", name);
+        git.commit(
+        ).setAuthor(
+                new PersonIdent(
+                        contents.getUserName(),
+                        contents.getUserEmail(),
+                        contents.getWhen(),
+                        TimeZone.getDefault()
+                )
+        ).setMessage(
+                contents.getCommitMessage()
+        ).call();
+        Log.info(
+                "[{}] Deleting files in directory: {}",
+                name,
+                contents.getDirectory().getAbsolutePath()
+        );
+        Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
+        return missingFiles;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/ProjectRepo.java

@@ -0,0 +1,24 @@
+package uk.ac.ic.wlgitbridge.bridge;
+
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface ProjectRepo {
+
+    String getProjectName();
+
+    Map<String, RawFile> getFiles() throws IOException, GitUserException;
+
+    Collection<String> commitAndGetMissing(
+            GitDirectoryContents gitDirectoryContents
+    ) throws IOException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java

@@ -1,13 +1,11 @@
 package uk.ac.ic.wlgitbridge.bridge;
 
 import com.google.api.client.auth.oauth2.Credential;
-import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.errors.RepositoryNotFoundException;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.transport.ServiceMayNotContinueException;
-import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
 
 import java.io.IOException;
 
@@ -18,15 +16,15 @@ public class WLBridgedProject {
 
     private final Repository repository;
     private final String name;
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
 
-    public WLBridgedProject(Repository repository, String name, BridgeAPI bridgeAPI) {
+    public WLBridgedProject(Repository repository, String name, Bridge bridgeAPI) {
         this.repository = repository;
         this.name = name;
         this.bridgeAPI = bridgeAPI;
     }
 
-    public void buildRepository(Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
+    public void buildRepository(Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
         bridgeAPI.lockForProject(name);
         try {
             if (repository.getObjectDatabase().exists()) {
@@ -42,21 +40,25 @@ public class WLBridgedProject {
         }
     }
 
-    private void updateRepositoryFromSnapshots(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
+    private void updateRepositoryFromSnapshots(
+            Credential oauth2,
+            Repository repository
+    ) throws RepositoryNotFoundException,
+             ServiceMayNotContinueException,
+             GitUserException {
         try {
-            bridgeAPI.getWritableRepositories(oauth2, name, repository);
+            bridgeAPI.getWritableRepositories(
+                    oauth2,
+                    new GitProjectRepo(repository, name)
+            );
         } catch (InvalidProjectException e) {
             throw new RepositoryNotFoundException(name);
-        } catch (SnapshotPostException e) {
-            throw new ServiceMayNotContinueException(e.getDescriptionLines().get(0), e);
-        } catch (GitAPIException e) {
-            throw new ServiceMayNotContinueException(e);
         } catch (IOException e) {
             throw new ServiceMayNotContinueException(e);
         }
     }
 
-    private void buildRepositoryFromScratch(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
+    private void buildRepositoryFromScratch(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
         if (!bridgeAPI.repositoryExists(oauth2, name)) {
             throw new RepositoryNotFoundException(name);
         }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java

@@ -0,0 +1,22 @@
+package uk.ac.ic.wlgitbridge.bridge.db;
+
+import java.util.List;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface DBStore {
+
+    List<String> getProjectNames();
+
+    void setLatestVersionForProject(String project, int versionID);
+
+    int getLatestVersionForProject(String project);
+
+    void addURLIndexForProject(String projectName, String url, String path);
+
+    void deleteFilesForProject(String project, String... files);
+
+    String getPathForURLInProject(String projectName, String url);
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java

@@ -1,4 +1,4 @@
-package uk.ac.ic.wlgitbridge.data.model.db;
+package uk.ac.ic.wlgitbridge.bridge.db;
 
 import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
 import uk.ac.ic.wlgitbridge.util.Log;
@@ -9,15 +9,15 @@ import java.util.Arrays;
 import java.util.List;
 
 /**
- * Created by Winston on 19/11/14.
+ * Created by winston on 20/08/2016.
  */
-public class SqlitePersistentStore implements PersistentStore {
+public class SqliteDBStore implements DBStore {
 
     private final SQLiteWLDatabase database;
 
-    public SqlitePersistentStore(File rootGitDirectory) {
+    public SqliteDBStore(File rootDirectory) {
         try {
-            database = new SQLiteWLDatabase(rootGitDirectory);
+            database = new SQLiteWLDatabase(rootDirectory);
         } catch (SQLException e) {
             throw new RuntimeException(e);
         } catch (ClassNotFoundException e) {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java

@@ -0,0 +1,12 @@
+package uk.ac.ic.wlgitbridge.bridge.lock;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface ProjectLock {
+    void lockAll();
+
+    void lockForProject(String projectName);
+
+    void unlockForProject(String projectName);
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java

@@ -0,0 +1,52 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class FSRepoStore implements RepoStore {
+
+    private final String repoStorePath;
+    private final File rootDirectory;
+
+    public FSRepoStore(String repoStorePath) {
+        this.repoStorePath = repoStorePath;
+        rootDirectory = initRootGitDirectory(repoStorePath);
+    }
+
+    @Override
+    public String getRepoStorePath() {
+        return repoStorePath;
+    }
+
+    @Override
+    public File getRootDirectory() {
+        return rootDirectory;
+    }
+
+    @Override
+    public void purgeNonexistentProjects(
+            Collection<String> existingProjectNames
+    ) {
+        List<String> excludedFromDeletion =
+                new ArrayList<>(existingProjectNames);
+        excludedFromDeletion.add(".wlgb");
+        Util.deleteInDirectoryApartFrom(
+                rootDirectory,
+                excludedFromDeletion.toArray(new String[] {})
+        );
+    }
+
+    private File initRootGitDirectory(String rootGitDirectoryPath) {
+        File rootGitDirectory = new File(rootGitDirectoryPath);
+        rootGitDirectory.mkdirs();
+        return rootGitDirectory;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java

@@ -0,0 +1,22 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+
+import java.io.File;
+import java.util.Collection;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface RepoStore {
+
+    String getRepoStorePath();
+
+    File getRootDirectory();
+
+
+    void purgeNonexistentProjects(
+            Collection<String> existingProjectNames
+    );
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java

@@ -0,0 +1,13 @@
+package uk.ac.ic.wlgitbridge.bridge.resource;
+
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface ResourceCache {
+    RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException;
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java

@@ -0,0 +1,107 @@
+package uk.ac.ic.wlgitbridge.bridge.resource;
+
+import com.ning.http.client.AsyncCompletionHandler;
+import com.ning.http.client.HttpResponseBodyPart;
+import com.ning.http.client.Response;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
+import uk.ac.ic.wlgitbridge.snapshot.base.Request;
+import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class UrlResourceCache implements ResourceCache {
+
+    private final DBStore dbStore;
+
+    public UrlResourceCache(DBStore dbStore) {
+        this.dbStore = dbStore;
+    }
+
+    @Override
+    public RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException {
+        String path = dbStore.getPathForURLInProject(projectName, url);
+        byte[] contents;
+        if (path == null) {
+            path = newPath;
+            contents = fetch(projectName, url, path);
+            fetchedUrls.put(url, contents);
+        } else {
+            Log.info("Found (" + projectName + "): " + url);
+            Log.info("At (" + projectName + "): " + path);
+            contents = fetchedUrls.get(url);
+            if (contents == null) {
+                RawFile rawFile = fileTable.get(path);
+                if (rawFile == null) {
+                    Log.warn(
+                            "File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
+                                    "File url is: " + url
+                    );
+                    contents = fetch(projectName, url, path);
+                } else {
+                    contents = rawFile.getContents();
+                }
+            }
+        }
+        return new RepositoryFile(newPath, contents);
+    }
+
+    private byte[] fetch(String projectName, final String url, String path) throws FailedConnectionException {
+        byte[] contents;
+        Log.info("GET -> " + url);
+        try {
+            contents = Request.httpClient.prepareGet(url).execute(new AsyncCompletionHandler<byte[]>() {
+
+                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+                @Override
+                public STATE onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
+                    bytes.write(bodyPart.getBodyPartBytes());
+                    return STATE.CONTINUE;
+                }
+
+                @Override
+                public byte[] onCompleted(Response response) throws Exception {
+                    byte[] data = bytes.toByteArray();
+                    bytes.close();
+                    Log.info(response.getStatusCode() + " " + response.getStatusText() + " (" + data.length + "B) -> " + url);
+                    return data;
+                }
+
+            }).get();
+        } catch (InterruptedException e) {
+            Log.warn(
+                    "Interrupted when fetching project: "  +
+                            projectName  +
+                            ", url: " +
+                            url +
+                            ", path: " +
+                            path,
+                    e
+            );
+            throw new FailedConnectionException();
+        } catch (ExecutionException e) {
+            Log.warn(
+                    "ExecutionException when fetching project: " +
+                            projectName +
+                            ", url: " +
+                            url +
+                            ", path: " +
+                            path,
+                    e
+            );
+            throw new FailedConnectionException();
+        }
+        dbStore.addURLIndexForProject(projectName, url, path);
+        return contents;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotAPI.java

@@ -0,0 +1,79 @@
+package uk.ac.ic.wlgitbridge.bridge.snapshot;
+
+import com.google.api.client.auth.oauth2.Credential;
+import uk.ac.ic.wlgitbridge.data.model.Snapshot;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
+
+import java.util.*;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class NetSnapshotAPI implements SnapshotAPI {
+
+    @Override
+    public Deque<Snapshot> getSnapshotsForProjectAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
+        List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(oauth2, projectName, version);
+        List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
+        LinkedList<Snapshot> snapshots = combine(snapshotInfos, snapshotDatas);
+        return snapshots;
+    }
+
+    private List<SnapshotInfo> getSnapshotInfosAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
+        SortedSet<SnapshotInfo> versions = new TreeSet<SnapshotInfo>();
+        GetDocRequest getDoc = new GetDocRequest(oauth2, projectName);
+        GetSavedVersRequest getSavedVers = new GetSavedVersRequest(oauth2, projectName);
+        getDoc.request();
+        getSavedVers.request();
+        GetDocResult latestDoc = getDoc.getResult();
+        int latest = latestDoc.getVersionID();
+        if (latest > version) {
+            for (SnapshotInfo snapshotInfo : getSavedVers.getResult().getSavedVers()) {
+                if (snapshotInfo.getVersionId() > version) {
+                    versions.add(snapshotInfo);
+                }
+            }
+            versions.add(new SnapshotInfo(latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
+
+        }
+        return new LinkedList<SnapshotInfo>(versions);
+    }
+
+    private List<SnapshotData> getMatchingSnapshotData(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) throws FailedConnectionException, ForbiddenException {
+        List<GetForVersionRequest> firedRequests = fireDataRequests(oauth2, projectName, snapshotInfos);
+        List<SnapshotData> snapshotDataList = new LinkedList<SnapshotData>();
+        for (GetForVersionRequest fired : firedRequests) {
+            snapshotDataList.add(fired.getResult().getSnapshotData());
+        }
+        return snapshotDataList;
+    }
+
+    private List<GetForVersionRequest> fireDataRequests(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
+        List<GetForVersionRequest> requests = new LinkedList<GetForVersionRequest>();
+        for (SnapshotInfo snapshotInfo : snapshotInfos) {
+            GetForVersionRequest request = new GetForVersionRequest(oauth2, projectName, snapshotInfo.getVersionId());
+            requests.add(request);
+            request.request();
+        }
+        return requests;
+    }
+
+    private LinkedList<Snapshot> combine(List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
+        LinkedList<Snapshot> snapshots = new LinkedList<Snapshot>();
+        Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
+        Iterator<SnapshotData> datas = snapshotDatas.iterator();
+        while (infos.hasNext()) {
+            snapshots.add(new Snapshot(infos.next(), datas.next()));
+        }
+        return snapshots;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotAPI.java

@@ -0,0 +1,21 @@
+package uk.ac.ic.wlgitbridge.bridge.snapshot;
+
+import com.google.api.client.auth.oauth2.Credential;
+import uk.ac.ic.wlgitbridge.data.model.Snapshot;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
+
+import java.util.Deque;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface SnapshotAPI {
+
+    Deque<Snapshot> getSnapshotsForProjectAfterVersion(
+            Credential oauth2,
+            String projectName,
+            int latestVersion
+    ) throws FailedConnectionException, GitUserException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java

@@ -0,0 +1,12 @@
+package uk.ac.ic.wlgitbridge.bridge.swap;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface SwapJob {
+
+    void start(int intervalMillis);
+
+    void stop();
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java

@@ -0,0 +1,53 @@
+package uk.ac.ic.wlgitbridge.bridge.swap;
+
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.util.Timer;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class SwapJobImpl implements SwapJob {
+
+    private final ProjectLock lock;
+    private final RepoStore repoStore;
+    private final SwapStore swapStore;
+    private final DBStore dbStore;
+
+    private final Timer timer;
+
+    public SwapJobImpl(
+            ProjectLock lock,
+            RepoStore repoStore,
+            DBStore dbStore, SwapStore swapStore
+    ) {
+
+        this.lock = lock;
+        this.repoStore = repoStore;
+        this.swapStore = swapStore;
+        this.dbStore = dbStore;
+        timer = new Timer();
+    }
+
+    @Override
+    public void start(int intervalMillis) {
+        timer.scheduleAtFixedRate(
+                Util.makeTimerTask(this::doSwap),
+                0,
+                intervalMillis
+        );
+    }
+
+    @Override
+    public void stop() {
+        timer.cancel();
+    }
+
+    private void doSwap() {
+        throw new UnsupportedOperationException();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java

@@ -0,0 +1,7 @@
+package uk.ac.ic.wlgitbridge.bridge.swap;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface SwapStore {
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLock.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java

@@ -1,5 +1,7 @@
 package uk.ac.ic.wlgitbridge.data;
 
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.locks.Lock;
@@ -9,7 +11,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 /**
  * Created by Winston on 20/11/14.
  */
-public class ProjectLock {
+public class ProjectLockImpl implements ProjectLock {
 
     private final Map<String, Lock> projectLocks;
     private final ReentrantReadWriteLock rwlock;
@@ -18,7 +20,7 @@ public class ProjectLock {
     private LockAllWaiter waiter;
     private boolean waiting;
 
-    public ProjectLock() {
+    public ProjectLockImpl() {
         projectLocks = new HashMap<String, Lock>();
         rwlock = new ReentrantReadWriteLock();
         rlock = rwlock.readLock();
@@ -26,6 +28,11 @@ public class ProjectLock {
         waiting = false;
     }
 
+    public ProjectLockImpl(LockAllWaiter waiter) {
+        this();
+        setWaiter(waiter);
+    }
+
     public void lockForProject(String projectName) {
         getLockForProjectName(projectName).lock();
         rlock.lock();

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ShutdownHook.java

@@ -1,30 +0,0 @@
-package uk.ac.ic.wlgitbridge.data;
-
-import uk.ac.ic.wlgitbridge.util.Log;
-
-/**
- * Created by Winston on 21/02/15.
- */
-public class ShutdownHook extends Thread implements LockAllWaiter {
-
-    private final ProjectLock projectLock;
-
-    public ShutdownHook(ProjectLock projectLock) {
-        this.projectLock = projectLock;
-        projectLock.setWaiter(this);
-    }
-
-    @Override
-    public void run() {
-        Log.info("Shutdown received.");
-        projectLock.lockAll();
-        Log.info("No projects to wait for.");
-        Log.info("Bye");
-    }
-
-    @Override
-    public void threadsRemaining(int threads) {
-        Log.info("Waiting for " + threads + " projects...");
-    }
-
-}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotFetcher.java

@@ -1,78 +1,10 @@
 package uk.ac.ic.wlgitbridge.data;
 
-import com.google.api.client.auth.oauth2.Credential;
-import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
-import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
-import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
-import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
-import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
-import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
-import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
-import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
-import uk.ac.ic.wlgitbridge.data.model.Snapshot;
-
-import java.util.*;
-
 /**
  * Created by Winston on 07/11/14.
  */
 public class SnapshotFetcher {
 
-    public LinkedList<Snapshot> getSnapshotsForProjectAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, SnapshotPostException, ForbiddenException {
-        List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(oauth2, projectName, version);
-        List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
-        LinkedList<Snapshot> snapshots = combine(snapshotInfos, snapshotDatas);
-        return snapshots;
-    }
-
-    private List<SnapshotInfo> getSnapshotInfosAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, SnapshotPostException, ForbiddenException {
-        SortedSet<SnapshotInfo> versions = new TreeSet<SnapshotInfo>();
-        GetDocRequest getDoc = new GetDocRequest(oauth2, projectName);
-        GetSavedVersRequest getSavedVers = new GetSavedVersRequest(oauth2, projectName);
-        getDoc.request();
-        getSavedVers.request();
-        GetDocResult latestDoc = getDoc.getResult();
-        int latest = latestDoc.getVersionID();
-        if (latest > version) {
-            for (SnapshotInfo snapshotInfo : getSavedVers.getResult().getSavedVers()) {
-                if (snapshotInfo.getVersionId() > version) {
-                    versions.add(snapshotInfo);
-                }
-            }
-            versions.add(new SnapshotInfo(latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
-
-        }
-        return new LinkedList<SnapshotInfo>(versions);
-    }
-
-    private List<SnapshotData> getMatchingSnapshotData(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) throws FailedConnectionException, ForbiddenException {
-        List<GetForVersionRequest> firedRequests = fireDataRequests(oauth2, projectName, snapshotInfos);
-        List<SnapshotData> snapshotDataList = new LinkedList<SnapshotData>();
-        for (GetForVersionRequest fired : firedRequests) {
-            snapshotDataList.add(fired.getResult().getSnapshotData());
-        }
-        return snapshotDataList;
-    }
-
-    private List<GetForVersionRequest> fireDataRequests(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
-        List<GetForVersionRequest> requests = new LinkedList<GetForVersionRequest>();
-        for (SnapshotInfo snapshotInfo : snapshotInfos) {
-            GetForVersionRequest request = new GetForVersionRequest(oauth2, projectName, snapshotInfo.getVersionId());
-            requests.add(request);
-            request.request();
-        }
-        return requests;
-    }
 
-    private LinkedList<Snapshot> combine(List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
-        LinkedList<Snapshot> snapshots = new LinkedList<Snapshot>();
-        Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
-        Iterator<SnapshotData> datas = snapshotDatas.iterator();
-        while (infos.hasNext()) {
-            snapshots.add(new Snapshot(infos.next(), datas.next()));
-        }
-        return snapshots;
-    }
 
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotRepositoryBuilder.java

@@ -5,9 +5,9 @@ import org.eclipse.jgit.errors.RepositoryNotFoundException;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
 import org.eclipse.jgit.transport.ServiceMayNotContinueException;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 import uk.ac.ic.wlgitbridge.bridge.WLBridgedProject;
-import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
 import uk.ac.ic.wlgitbridge.util.Log;
 
@@ -19,13 +19,13 @@ import java.io.IOException;
  */
 public class SnapshotRepositoryBuilder {
 
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
 
-    public SnapshotRepositoryBuilder(BridgeAPI bridgeAPI) {
+    public SnapshotRepositoryBuilder(Bridge bridgeAPI) {
         this.bridgeAPI = bridgeAPI;
     }
 
-    public Repository getRepositoryWithNameAtRootDirectory(String name, File rootDirectory, Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, ForbiddenException {
+    public Repository getRepositoryWithNameAtRootDirectory(String name, File rootDirectory, Credential oauth2) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
         if (!bridgeAPI.repositoryExists(oauth2, name)) {
             throw new RepositoryNotFoundException(name);
         }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java

@@ -1,183 +0,0 @@
-package uk.ac.ic.wlgitbridge.data.model;
-
-import com.google.api.client.auth.oauth2.Credential;
-import org.eclipse.jgit.api.Git;
-import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.lib.PersonIdent;
-import org.eclipse.jgit.lib.Repository;
-import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
-import uk.ac.ic.wlgitbridge.data.SnapshotFetcher;
-import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
-import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
-import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
-import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
-import uk.ac.ic.wlgitbridge.data.model.db.SqlitePersistentStore;
-import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
-import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
-import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
-import uk.ac.ic.wlgitbridge.util.Log;
-import uk.ac.ic.wlgitbridge.util.Util;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-
-/**
- * Created by Winston on 06/11/14.
- */
-public class DataStore {
-
-    private final File rootGitDirectory;
-    private final PersistentStore persistentStore;
-    private final SnapshotFetcher snapshotFetcher;
-    private final ResourceFetcher resourceFetcher;
-
-    public DataStore(String rootGitDirectoryPath) {
-        rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath);
-        persistentStore = new SqlitePersistentStore(rootGitDirectory);
-        List<String> excludedFromDeletion = persistentStore.getProjectNames();
-        excludedFromDeletion.add(".wlgb");
-        Util.deleteInDirectoryApartFrom(
-                rootGitDirectory,
-                excludedFromDeletion.toArray(new String[] {})
-        );
-
-        snapshotFetcher = new SnapshotFetcher();
-        resourceFetcher = new ResourceFetcher(persistentStore);
-    }
-
-    public void updateProjectWithName(Credential oauth2,
-                                      String name,
-                                      Repository repository)
-            throws IOException,
-                   SnapshotPostException,
-                   GitAPIException,
-                   ForbiddenException {
-        LinkedList<Snapshot> snapshots =
-                snapshotFetcher.getSnapshotsForProjectAfterVersion(
-                        oauth2,
-                        name,
-                        persistentStore.getLatestVersionForProject(name)
-                );
-
-        makeCommitsFromSnapshots(name, repository, snapshots);
-
-        if (!snapshots.isEmpty()) {
-            persistentStore.setLatestVersionForProject(
-                    name,
-                    snapshots.getLast().getVersionID()
-            );
-        }
-    }
-
-    private void makeCommitsFromSnapshots(String name,
-                                          Repository repository,
-                                          List<Snapshot> snapshots)
-            throws IOException, GitAPIException, SnapshotPostException {
-        for (Snapshot snapshot : snapshots) {
-            Map<String, RawFile> fileTable = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable();
-            List<RawFile> files = new LinkedList<RawFile>();
-            files.addAll(snapshot.getSrcs());
-            Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
-            for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
-                files.add(
-                        resourceFetcher.get(
-                                name,
-                                snapshotAttachment.getUrl(),
-                                snapshotAttachment.getPath(),
-                                fileTable,
-                                fetchedUrls
-                        )
-                );
-            }
-            Log.info(
-                    "[{}] Committing version ID: {}",
-                    name,
-                    snapshot.getVersionID()
-            );
-            commit(name,
-                    new GitDirectoryContents(
-                            files,
-                            rootGitDirectory,
-                            name,
-                            snapshot),
-                    repository);
-        }
-    }
-
-    private void commit(String name,
-                        GitDirectoryContents contents,
-                        Repository repository) throws IOException,
-                                                      GitAPIException {
-        Log.info("[{}] Writing commit", name);
-        contents.write();
-        Git git = new Git(repository);
-        Log.info("[{}] Getting missing files", name);
-        Set<String> missingFiles = git.status().call().getMissing();
-        for (String missing : missingFiles) {
-            Log.info("[{}] Git rm {}", name, missing);
-            git.rm().setCached(true).addFilepattern(missing).call();
-        }
-        Log.info("[{}] Calling Git add", name);
-        git.add().addFilepattern(".").call();
-        Log.info("[{}] Calling Git commit", name);
-        git.commit(
-        ).setAuthor(
-                new PersonIdent(
-                        contents.getUserName(),
-                        contents.getUserEmail(),
-                        contents.getWhen(),
-                        TimeZone.getDefault()
-                )
-        ).setMessage(
-                contents.getCommitMessage()
-        ).call();
-        persistentStore.deleteFilesForProject(
-                name,
-                missingFiles.toArray(new String[missingFiles.size()])
-        );
-        Log.info(
-                "[{}] Deleting files in directory: {}",
-                name,
-                contents.getDirectory().getAbsolutePath()
-        );
-        Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
-    }
-
-    public CandidateSnapshot
-    createCandidateSnapshot(String projectName,
-                            RawDirectory directoryContents,
-                            RawDirectory oldDirectoryContents)
-            throws SnapshotPostException,
-                   IOException {
-        CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
-                projectName,
-                persistentStore.getLatestVersionForProject(projectName),
-                directoryContents,
-                oldDirectoryContents
-        );
-        candidateSnapshot.writeServletFiles(rootGitDirectory);
-        return candidateSnapshot;
-    }
-
-    public void approveSnapshot(int versionID,
-                                CandidateSnapshot candidateSnapshot) {
-        List<String> deleted = candidateSnapshot.getDeleted();
-        persistentStore.setLatestVersionForProject(
-                candidateSnapshot.getProjectName(),
-                versionID
-        );
-        persistentStore.deleteFilesForProject(
-                candidateSnapshot.getProjectName(),
-                deleted.toArray(new String[deleted.size()])
-        );
-    }
-
-    private File initRootGitDirectory(String rootGitDirectoryPath) {
-        File rootGitDirectory = new File(rootGitDirectoryPath);
-        rootGitDirectory.mkdirs();
-        return rootGitDirectory;
-    }
-
-}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java

@@ -1,108 +1,9 @@
 package uk.ac.ic.wlgitbridge.data.model;
 
-import com.ning.http.client.AsyncCompletionHandler;
-import com.ning.http.client.HttpResponseBodyPart;
-import com.ning.http.client.Response;
-import org.eclipse.jgit.lib.Repository;
-import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
-import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
-import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
-import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
-import uk.ac.ic.wlgitbridge.snapshot.base.Request;
-import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
-import uk.ac.ic.wlgitbridge.util.Log;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
 /**
  * Created by Winston on 21/02/15.
  */
 public class ResourceFetcher {
 
-    private final PersistentStore persistentStore;
-
-    public ResourceFetcher(PersistentStore persistentStore) {
-        this.persistentStore = persistentStore;
-    }
-
-    public RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException, SnapshotPostException {
-        String path = persistentStore.getPathForURLInProject(projectName, url);
-        byte[] contents;
-        if (path == null) {
-            path = newPath;
-            contents = fetch(projectName, url, path);
-            fetchedUrls.put(url, contents);
-        } else {
-            Log.info("Found (" + projectName + "): " + url);
-            Log.info("At (" + projectName + "): " + path);
-            contents = fetchedUrls.get(url);
-            if (contents == null) {
-                RawFile rawFile = fileTable.get(path);
-                if (rawFile == null) {
-                    Log.warn(
-                        "File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
-                        "File url is: " + url
-                    );
-                    contents = fetch(projectName, url, path);
-                } else {
-                    contents = rawFile.getContents();
-                }
-            }
-        }
-        return new RepositoryFile(newPath, contents);
-    }
-
-    private byte[] fetch(String projectName, final String url, String path) throws FailedConnectionException {
-        byte[] contents;
-        Log.info("GET -> " + url);
-        try {
-            contents = Request.httpClient.prepareGet(url).execute(new AsyncCompletionHandler<byte[]>() {
-
-                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-
-                @Override
-                public STATE onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
-                    bytes.write(bodyPart.getBodyPartBytes());
-                    return STATE.CONTINUE;
-                }
-
-                @Override
-                public byte[] onCompleted(Response response) throws Exception {
-                    byte[] data = bytes.toByteArray();
-                    bytes.close();
-                    Log.info(response.getStatusCode() + " " + response.getStatusText() + " (" + data.length + "B) -> " + url);
-                    return data;
-                }
 
-            }).get();
-        } catch (InterruptedException e) {
-            Log.warn(
-                    "Interrupted when fetching project: "  +
-                            projectName  +
-                            ", url: " +
-                            url +
-                            ", path: " +
-                            path,
-                    e
-            );
-            throw new FailedConnectionException();
-        } catch (ExecutionException e) {
-            Log.warn(
-                    "ExecutionException when fetching project: " +
-                            projectName +
-                            ", url: " +
-                            url +
-                            ", path: " +
-                            path,
-                    e
-            );
-            throw new FailedConnectionException();
-        }
-        persistentStore.addURLIndexForProject(projectName, url, path);
-        return contents;
-    }
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/GitUserException.java

@@ -0,0 +1,13 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import java.util.List;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public abstract class GitUserException extends Exception {
+
+    public abstract String getMessage();
+    public abstract List<String> getDescriptionLines();
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SizeLimitExceededException.java

@@ -1,13 +1,11 @@
 package uk.ac.ic.wlgitbridge.git.exception;
 
-import com.google.gson.JsonElement;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
 import uk.ac.ic.wlgitbridge.util.Util;
 
 import java.util.Arrays;
 import java.util.List;
 
-public class SizeLimitExceededException extends SnapshotPostException {
+public class SizeLimitExceededException extends GitUserException {
 
     private static String path = null;
 
@@ -26,12 +24,8 @@ public class SizeLimitExceededException extends SnapshotPostException {
         String filename = path != null ? "File '" + path + "' is" : "There's a file";
         return Arrays.asList(
             filename + " too large to push to " + Util.getServiceName() + " via git",
-            "the recommended maximum file size is 15MiB"
+            "the recommended maximum file size is 50 MiB"
         );
     }
 
-    @Override
-    public void fromJSON(JsonElement json) {
-
-    }
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SnapshotAPIException.java

@@ -0,0 +1,11 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public abstract class SnapshotAPIException
+        extends GitUserException
+        implements JSONSource {
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java

@@ -6,7 +6,7 @@ import org.eclipse.jgit.transport.ReceivePack;
 import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
 import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
 import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
 import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
 import uk.ac.ic.wlgitbridge.util.Util;
@@ -19,9 +19,9 @@ import javax.servlet.http.HttpServletRequest;
 /* */
 public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
 
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
 
-    public WLReceivePackFactory(BridgeAPI bridgeAPI) {
+    public WLReceivePackFactory(Bridge bridgeAPI) {
         this.bridgeAPI = bridgeAPI;
     }
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java

@@ -8,6 +8,7 @@ import org.eclipse.jgit.transport.resolver.RepositoryResolver;
 import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
 import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
 import uk.ac.ic.wlgitbridge.data.SnapshotRepositoryBuilder;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
 import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
 import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
@@ -51,6 +52,8 @@ public class WLRepositoryResolver implements RepositoryResolver<HttpServletReque
             throw new ServiceMayNotContinueException(e);
         } catch (ForbiddenException e) {
             throw new ServiceNotAuthorizedException();
+        } catch (GitUserException e) {
+            throw new ServiceMayNotContinueException(e.getMessage(), e);
         }
     }
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java

@@ -6,12 +6,12 @@ import org.eclipse.jgit.transport.PreReceiveHook;
 import org.eclipse.jgit.transport.ReceiveCommand;
 import org.eclipse.jgit.transport.ReceiveCommand.Result;
 import org.eclipse.jgit.transport.ReceivePack;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
 import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
 import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
-import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@@ -26,11 +26,11 @@ import java.util.Iterator;
  */
 public class WriteLatexPutHook implements PreReceiveHook {
 
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
     private final String hostname;
     private final Credential oauth2;
 
-    public WriteLatexPutHook(BridgeAPI bridgeAPI, String hostname, Credential oauth2) {
+    public WriteLatexPutHook(Bridge bridgeAPI, String hostname, Credential oauth2) {
         this.bridgeAPI = bridgeAPI;
         this.hostname = hostname;
         this.oauth2 = oauth2;
@@ -72,7 +72,7 @@ public class WriteLatexPutHook implements PreReceiveHook {
         receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
     }
 
-    private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, SnapshotPostException, ForbiddenException {
+    private void handleReceiveCommand(Credential oauth2, Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
         checkBranch(receiveCommand);
         checkForcedPush(receiveCommand);
         bridgeAPI.putDirectoryContentsToProjectWithName(
@@ -97,13 +97,13 @@ public class WriteLatexPutHook implements PreReceiveHook {
         }
     }
 
-    private RawDirectory getPushedDirectoryContents(Repository repository, ReceiveCommand receiveCommand) throws IOException, SnapshotPostException {
+    private RawDirectory getPushedDirectoryContents(Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
         return new RepositoryObjectTreeWalker(repository,
                                               receiveCommand.getNewId())
                .getDirectoryContents();
     }
 
-    private RawDirectory getOldDirectoryContents(Repository repository) throws IOException, SnapshotPostException {
+    private RawDirectory getOldDirectoryContents(Repository repository) throws IOException, GitUserException {
         return new RepositoryObjectTreeWalker(repository).getDirectoryContents();
     }
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java

@@ -7,7 +7,7 @@ import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
 import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
 import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
 import uk.ac.ic.wlgitbridge.data.SnapshotRepositoryBuilder;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 
 import javax.servlet.ServletException;
 
@@ -16,7 +16,7 @@ import javax.servlet.ServletException;
  */
 public class WLGitServlet extends GitServlet {
 
-    public WLGitServlet(ServletContextHandler servletContextHandler, BridgeAPI bridgeAPI, String rootGitDirectoryPath) throws ServletException, InvalidRootDirectoryPathException {
+    public WLGitServlet(ServletContextHandler servletContextHandler, Bridge bridgeAPI, String rootGitDirectoryPath) throws ServletException, InvalidRootDirectoryPathException {
         setRepositoryResolver(new WLRepositoryResolver(rootGitDirectoryPath, new SnapshotRepositoryBuilder(bridgeAPI)));
         setReceivePackFactory(new WLReceivePackFactory(bridgeAPI));
         setUploadPackFactory(new WLUploadPackFactory());

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java

@@ -8,8 +8,8 @@ import org.eclipse.jgit.treewalk.TreeWalk;
 import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
 import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -36,7 +36,7 @@ public class RepositoryObjectTreeWalker {
         this(repository, repository.resolve("HEAD~" + fromHead));
     }
 
-    public RawDirectory getDirectoryContents() throws IOException, SnapshotPostException {
+    public RawDirectory getDirectoryContents() throws IOException, GitUserException {
         return new RawDirectory(walkGitObjectTree());
     }
 
@@ -51,7 +51,7 @@ public class RepositoryObjectTreeWalker {
         return treeWalk;
     }
 
-    private Map<String, RawFile> walkGitObjectTree() throws IOException, SnapshotPostException {
+    private Map<String, RawFile> walkGitObjectTree() throws IOException, GitUserException {
         Map<String, RawFile> fileContentsTable = new HashMap<String, RawFile>();
         if (treeWalk == null) {
             return fileContentsTable;

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/FileHandler.java

@@ -4,7 +4,7 @@ import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.ResourceHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidPostbackKeyException;
 
 import javax.servlet.ServletException;
@@ -22,10 +22,10 @@ import java.util.regex.Pattern;
 public class FileHandler extends ResourceHandler {
     private static final Logger LOG = LoggerFactory.getLogger(FileHandler.class);
 
-    private final BridgeAPI writeLatexDataSource;
+    private final Bridge writeLatexDataSource;
     private final Pattern DOC_KEY_PATTERN = Pattern.compile("^/(\\w+)/.+$");
 
-    public FileHandler(BridgeAPI writeLatexDataSource) {
+    public FileHandler(Bridge writeLatexDataSource) {
         this.writeLatexDataSource = writeLatexDataSource;
     }
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java

@@ -8,7 +8,12 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import uk.ac.ic.wlgitbridge.application.config.Config;
 import uk.ac.ic.wlgitbridge.application.jetty.NullLogger;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.db.SqliteDBStore;
+import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
 import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
 import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
 import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest;
@@ -31,7 +36,7 @@ import java.util.EnumSet;
  */
 public class GitBridgeServer {
 
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
 
     private final Server jettyServer;
 
@@ -43,7 +48,14 @@ public class GitBridgeServer {
         org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
         this.port = config.getPort();
         this.rootGitDirectoryPath = config.getRootGitDirectory();
-        bridgeAPI = new BridgeAPI(rootGitDirectoryPath);
+        RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
+        DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
+        SwapStore swapStore = new SwapStore() {};
+        bridgeAPI = Bridge.make(
+                repoStore,
+                dbStore,
+                swapStore
+        );
         jettyServer = new Server(port);
         configureJettyServer(config);
         SnapshotAPIRequest.setBasicAuth(config.getUsername(), config.getPassword());

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/PostbackContents.java

@@ -3,7 +3,7 @@ package uk.ac.ic.wlgitbridge.server;
 import com.google.gson.Gson;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
 import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.UnexpectedPostbackException;
@@ -17,7 +17,7 @@ public class PostbackContents implements JSONSource {
 
     private static final String CODE_SUCCESS = "upToDate";
 
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
     private final String projectName;
     private final String postbackKey;
 
@@ -26,7 +26,7 @@ public class PostbackContents implements JSONSource {
     private int versionID;
     private SnapshotPostException exception;
 
-    public PostbackContents(BridgeAPI bridgeAPI, String projectName, String postbackKey, String contents) {
+    public PostbackContents(Bridge bridgeAPI, String projectName, String postbackKey, String contents) {
         this.bridgeAPI = bridgeAPI;
         this.projectName = projectName;
         this.postbackKey = postbackKey;

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/PostbackHandler.java

@@ -4,7 +4,7 @@ import com.google.gson.JsonObject;
 import com.google.gson.JsonPrimitive;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.AbstractHandler;
-import uk.ac.ic.wlgitbridge.bridge.BridgeAPI;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.UnexpectedPostbackException;
 import uk.ac.ic.wlgitbridge.util.Log;
 import uk.ac.ic.wlgitbridge.util.Util;
@@ -19,9 +19,9 @@ import java.io.IOException;
  */
 public class PostbackHandler extends AbstractHandler {
 
-    private final BridgeAPI bridgeAPI;
+    private final Bridge bridgeAPI;
 
-    public PostbackHandler(BridgeAPI bridgeAPI) {
+    public PostbackHandler(Bridge bridgeAPI) {
         this.bridgeAPI = bridgeAPI;
     }
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/ForbiddenException.java

@@ -1,7 +1,28 @@
 package uk.ac.ic.wlgitbridge.snapshot.base;
 
+import com.google.gson.JsonElement;
+import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
+
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * Created by winston on 25/10/15.
  */
-public class ForbiddenException extends Throwable {
+public class ForbiddenException extends SnapshotAPIException {
+    @Override
+    public void fromJSON(JsonElement json) {
+
+    }
+
+    @Override
+    public String getMessage() {
+        return "forbidden";
+    }
+
+    @Override
+    public List<String> getDescriptionLines() {
+        return Arrays.asList(getMessage());
+    }
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java

@@ -2,14 +2,14 @@ package uk.ac.ic.wlgitbridge.snapshot.getdoc;
 
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
 import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.base.Request;
 import uk.ac.ic.wlgitbridge.snapshot.base.Result;
 import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
 import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
 import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.WLUser;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
-import uk.ac.ic.wlgitbridge.snapshot.base.Request;
-import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.ProtectedProjectException;
 
 /**
  * Created by Winston on 06/11/14.
@@ -21,7 +21,7 @@ public class GetDocResult extends Result {
     private String createdAt;
     private WLUser user;
 
-    private SnapshotPostException exception;
+    private SnapshotAPIException exception;
     private ForbiddenException forbidden;
 
     public GetDocResult(Request request, JsonElement json) throws FailedConnectionException {
@@ -94,7 +94,7 @@ public class GetDocResult extends Result {
         }
     }
 
-    public int getVersionID() throws SnapshotPostException {
+    public int getVersionID() throws GitUserException {
         if (exception != null) {
             throw exception;
         }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/exception/InvalidProjectException.java

@@ -2,8 +2,7 @@ package uk.ac.ic.wlgitbridge.snapshot.getdoc.exception;
 
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
+import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -11,14 +10,10 @@ import java.util.List;
 /**
  * Created by Winston on 08/11/14.
  */
-public class InvalidProjectException extends SnapshotPostException {
+public class InvalidProjectException extends SnapshotAPIException {
 
     private List<String> errors;
 
-    public InvalidProjectException(JsonObject json) {
-        super(json);
-    }
-
     public InvalidProjectException() {
         super();
         errors = new LinkedList<String>();
@@ -37,7 +32,8 @@ public class InvalidProjectException extends SnapshotPostException {
     @Override
     public void fromJSON(JsonElement json) {
         errors = new LinkedList<String>();
-        JsonArray errors = json.getAsJsonObject().get("errors").getAsJsonArray();
+        JsonArray errors =
+                json.getAsJsonObject().get("errors").getAsJsonArray();
         for (JsonElement error : errors) {
             this.errors.add(error.getAsString());
         }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/push/exception/SnapshotPostException.java

@@ -1,14 +1,12 @@
 package uk.ac.ic.wlgitbridge.snapshot.push.exception;
 
 import com.google.gson.JsonElement;
-import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
-
-import java.util.List;
+import uk.ac.ic.wlgitbridge.git.exception.SnapshotAPIException;
 
 /**
  * Created by Winston on 16/11/14.
  */
-public abstract class SnapshotPostException extends Exception implements JSONSource {
+public abstract class SnapshotPostException extends SnapshotAPIException {
 
     public SnapshotPostException() {
 
@@ -18,7 +16,4 @@ public abstract class SnapshotPostException extends Exception implements JSONSou
         fromJSON(jsonElement);
     }
 
-    public abstract String getMessage();
-    public abstract List<String> getDescriptionLines();
-
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java

@@ -19,6 +19,15 @@ public class Util {
     private static String POSTBACK_URL;
     private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
 
+    public static TimerTask makeTimerTask(Runnable lamb) {
+        return new TimerTask() {
+            @Override
+            public void run() {
+                lamb.run();
+            }
+        };
+    }
+
     public static String entries(int entries) {
         if (entries == 1) {
             return "entry";

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java

@@ -0,0 +1,58 @@
+package uk.ac.ic.wlgitbridge.bridge;
+
+import org.junit.Before;
+import org.junit.Test;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
+import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
+import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class BridgeTest {
+
+    private Bridge bridge;
+
+    private ProjectLock lock;
+    private RepoStore repoStore;
+    private DBStore dbStore;
+    private SwapStore swapStore;
+    private SnapshotAPI snapshotAPI;
+    private ResourceCache resourceCache;
+    private SwapJob swapJob;
+
+    @Before
+    public void setup() {
+        lock = mock(ProjectLock.class);
+        repoStore = mock(RepoStore.class);
+        dbStore = mock(DBStore.class);
+        swapStore = mock(SwapStore.class);
+        snapshotAPI = mock(SnapshotAPI.class);
+        swapJob = mock(SwapJob.class);
+        bridge = new Bridge(
+                lock,
+                repoStore,
+                dbStore,
+                swapStore,
+                snapshotAPI,
+                resourceCache,
+                swapJob
+        );
+    }
+
+    @Test
+    public void shutdownStopsSwapJob() {
+        bridge.startSwapJob(1000);
+        bridge.doShutdown();
+        verify(swapJob).start(1000);
+        verify(swapJob).stop();
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java

@@ -9,10 +9,12 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.mockserver.client.server.MockServerClient;
 import org.mockserver.junit.MockServerRule;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
+import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
-import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
-import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -32,7 +34,7 @@ public class ResourceFetcherTest {
     private MockServerClient mockServerClient;
 
     @Test
-    public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException, SnapshotPostException {
+    public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException, GitUserException {
         final String testProjectName = "123abc";
         final String testUrl = "http://localhost:" + mockServerRule.getPort() + "/123abc";
         final String oldTestPath = "testPath";
@@ -50,23 +52,23 @@ public class ResourceFetcherTest {
         );
 
         final Mockery context = new Mockery();
-        final PersistentStore persistentStore = context.mock(PersistentStore.class);
+        final DBStore dbStore = context.mock(DBStore.class);
         context.checking(new Expectations() {{
             // It should fetch the file once it finds it is missing.
-            oneOf(persistentStore).getPathForURLInProject(testProjectName, testUrl);
+            oneOf(dbStore).getPathForURLInProject(testProjectName, testUrl);
             will(returnValue(oldTestPath));
 
             // It should update the URL index store once it has fetched; at present, it does not actually change the stored path.
-            oneOf(persistentStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
+            oneOf(dbStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
         }});
 
-        ResourceFetcher resourceFetcher = new ResourceFetcher(persistentStore);
+        ResourceCache resources = new UrlResourceCache(dbStore);
         TemporaryFolder repositoryFolder = new TemporaryFolder();
         repositoryFolder.create();
         Repository repository = new FileRepositoryBuilder().setWorkTree(repositoryFolder.getRoot()).build();
         Map<String, RawFile> fileTable = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable();
         Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
-        resourceFetcher.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
+        resources.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
 
         // We don't bother caching in this case, at present.
         assertEquals(0, fetchedUrls.size());

services/git-bridge/writelatex-git-bridge.iml

@@ -13,7 +13,7 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
-    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock-junit4:2.8.2" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock:2.8.2" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock-testjar:2.8.2" level="project" />
@@ -107,5 +107,7 @@
     <orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcprov-jdk15on:1.52" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: janino:janino:2.5.10" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.mock-server:mockserver-logging:3.10.4" level="project" />
+    <orderEntry type="library" name="Maven: org.mockito:mockito-core:1.10.19" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: org.objenesis:objenesis:2.1" level="project" />
   </component>
 </module>