Fully implement max file size

- add `repoStore.maxFileSize` key to config
- use maxFileSize in ResourceCache on both header path and blob
  download path
- make failures during commit less fragile

services/git-bridge/conf/example_config.json

@@ -11,6 +11,9 @@
         "oauth2ClientSecret": "asdf",
         "oauth2Server": "https://localhost"
     },
+    "repoStore": {
+        "maxFileSize": 52428800
+    },
     "swapStore": {
         "type": "s3",
         "awsAccessKey": "asdf",

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java

@@ -4,6 +4,7 @@ import com.google.gson.Gson;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
 import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
 import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
 import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
@@ -30,6 +31,7 @@ public class Config implements JSONSource {
                 config.postbackURL,
                 config.serviceName,
                 Oauth2.asSanitised(config.oauth2),
+                config.repoStore,
                 SwapStoreConfig.sanitisedCopy(config.swapStore),
                 config.swapJob
         );
@@ -45,6 +47,8 @@ public class Config implements JSONSource {
     @Nullable
     private Oauth2 oauth2;
     @Nullable
+    private RepoStoreConfig repoStore;
+    @Nullable
     private SwapStoreConfig swapStore;
     @Nullable
     private SwapJobConfig swapJob;
@@ -69,6 +73,7 @@ public class Config implements JSONSource {
             String postbackURL,
             String serviceName,
             Oauth2 oauth2,
+            RepoStoreConfig repoStore,
             SwapStoreConfig swapStore,
             SwapJobConfig swapJob
     ) {
@@ -80,6 +85,7 @@ public class Config implements JSONSource {
         this.postbackURL = postbackURL;
         this.serviceName = serviceName;
         this.oauth2 = oauth2;
+        this.repoStore = repoStore;
         this.swapStore = swapStore;
         this.swapJob = swapJob;
     }
@@ -108,6 +114,8 @@ public class Config implements JSONSource {
             postbackURL += "/";
         }
         oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
+        repoStore = new Gson().fromJson(
+                configObject.get("repoStore"), RepoStoreConfig.class);
         swapStore = new Gson().fromJson(
                 configObject.get("swapStore"),
                 SwapStoreConfig.class
@@ -161,6 +169,10 @@ public class Config implements JSONSource {
         return oauth2;
     }
 
+    public Optional<RepoStoreConfig> getRepoStore() {
+        return Optional.ofNullable(repoStore);
+    }
+
     public Optional<SwapStoreConfig> getSwapStore() {
         return Optional.ofNullable(swapStore);
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java

@@ -2,6 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge;
 
 import com.google.api.client.auth.oauth2.Credential;
 import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import uk.ac.ic.wlgitbridge.application.config.Config;
 import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
 import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
@@ -9,20 +10,16 @@ import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
 import uk.ac.ic.wlgitbridge.bridge.gc.GcJobImpl;
 import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
 import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
-import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
-import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo;
-import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
-import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.repo.*;
 import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
 import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
 import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi;
 import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
 import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
-import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
 import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobImpl;
 import uk.ac.ic.wlgitbridge.bridge.swap.store.S3SwapStore;
 import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
-import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
 import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
 import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
 import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
@@ -118,6 +115,7 @@ import java.util.*;
  *
  * 6. The Snapshot API, which provides data from the Overleaf app.
  *
+ *    @see SnapshotApiFacade - wraps a concrete instance of the Snapshot API.
  *    @see SnapshotApi - the interface for the Snapshot API.
  *    @see NetSnapshotApi - the default concrete implementation
  *
@@ -139,6 +137,8 @@ import java.util.*;
  */
 public class Bridge {
 
+    private final Config config;
+
     private final ProjectLock lock;
 
     private final RepoStore repoStore;
@@ -157,29 +157,31 @@ public class Bridge {
      * swap store, and the swap job config.
      *
      * This should be the method used to create a Bridge.
+     * @param config The config to use
      * @param repoStore The repo store to use
      * @param dbStore The db store to use
      * @param swapStore The swap store to use
-     * @param swapJobConfig The swap config to use, or empty for no-op
+     * @param snapshotApi The snapshot api to use
      * @return The constructed Bridge.
      */
     public static Bridge make(
+            Config config,
             RepoStore repoStore,
             DBStore dbStore,
             SwapStore swapStore,
-            Optional<SwapJobConfig> swapJobConfig,
             SnapshotApi snapshotApi
     ) {
         ProjectLock lock = new ProjectLockImpl((int threads) ->
                 Log.info("Waiting for " + threads + " projects...")
         );
         return new Bridge(
+                config,
                 lock,
                 repoStore,
                 dbStore,
                 swapStore,
                 SwapJob.fromConfig(
-                        swapJobConfig,
+                        config.getSwapJob(),
                         lock,
                         repoStore,
                         dbStore,
@@ -205,6 +207,7 @@ public class Bridge {
      * @param resourceCache the {@link ResourceCache} to use
      */
     Bridge(
+            Config config,
             ProjectLock lock,
             RepoStore repoStore,
             DBStore dbStore,
@@ -214,6 +217,7 @@ public class Bridge {
             SnapshotApiFacade snapshotAPI,
             ResourceCache resourceCache
     ) {
+        this.config = config;
         this.lock = lock;
         this.repoStore = repoStore;
         this.dbStore = dbStore;
@@ -621,6 +625,11 @@ public class Bridge {
 
         makeCommitsFromSnapshots(repo, snapshots);
 
+        // TODO: in case crashes around here, add an
+        // "updating_from_commit" column to the DB as a way to rollback the
+        // any failed partial updates before re-trying
+        // Also need to consider the empty state (a new git init'd repo being
+        // the rollback target)
         if (!snapshots.isEmpty()) {
             dbStore.setLatestVersionForProject(
                     projectName,
@@ -635,7 +644,7 @@ public class Bridge {
      * Performs the actual Git commits on the disk.
      *
      * Each commit adds files to the db store
-     * ({@link ResourceCache#get(String, String, String, Map, Map)},
+     * ({@link ResourceCache#get(String, String, String, Map, Map, Optional)},
      * and then removes any files that were deleted.
      * @param repo The repository to commit to
      * @param snapshots The snapshots to commit
@@ -647,10 +656,25 @@ public class Bridge {
             Collection<Snapshot> snapshots
     ) throws IOException, GitUserException {
         String name = repo.getProjectName();
+        Optional<Long> maxSize = config
+                .getRepoStore()
+                .flatMap(RepoStoreConfig::getMaxFileSize);
         for (Snapshot snapshot : snapshots) {
-            Map<String, RawFile> fileTable = repo.getFiles();
-            List<RawFile> files = new LinkedList<>();
+            RawDirectory directory = repo.getDirectory();
+            Map<String, RawFile> fileTable = directory.getFileTable();
+            List<RawFile> files = new ArrayList<>();
             files.addAll(snapshot.getSrcs());
+            for (RawFile file : files) {
+                long size = file.size();
+                /* Can't throw in ifPresent... */
+                if (maxSize.isPresent()) {
+                    long maxSize_ = maxSize.get();
+                    if (size >= maxSize_) {
+                        throw new SizeLimitExceededException(
+                                Optional.of(file.getPath()), size, maxSize_);
+                    }
+                }
+            }
             Map<String, byte[]> fetchedUrls = new HashMap<>();
             for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
                 files.add(
@@ -659,7 +683,8 @@ public class Bridge {
                                 snapshotAttachment.getUrl(),
                                 snapshotAttachment.getPath(),
                                 fileTable,
-                                fetchedUrls
+                                fetchedUrls,
+                                maxSize
                         )
                 );
             }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java

@@ -2,6 +2,8 @@ package uk.ac.ic.wlgitbridge.bridge.repo;
 
 import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import org.apache.commons.io.FileUtils;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
 import uk.ac.ic.wlgitbridge.util.Project;
 import uk.ac.ic.wlgitbridge.util.Tar;
 
@@ -12,6 +14,7 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Optional;
 import java.util.function.Function;
 
 import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
@@ -21,17 +24,35 @@ import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
  */
 public class FSGitRepoStore implements RepoStore {
 
+    private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
+
     private final String repoStorePath;
+
     private final File rootDirectory;
+
+    private final long maxFileSize;
+
     private final Function<File, Long> fsSizer;
 
-    public FSGitRepoStore(String repoStorePath) {
-        this(repoStorePath, d -> d.getTotalSpace() - d.getFreeSpace());
+    public FSGitRepoStore(
+            String repoStorePath,
+            Optional<Long> maxFileSize
+    ) {
+        this(
+                repoStorePath,
+                maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
+                d -> d.getTotalSpace() - d.getFreeSpace()
+        );
     }
 
-    public FSGitRepoStore(String repoStorePath, Function<File, Long> fsSizer) {
+    public FSGitRepoStore(
+            String repoStorePath,
+            long maxFileSize,
+            Function<File, Long> fsSizer
+    ) {
         this.repoStorePath = repoStorePath;
         rootDirectory = initRootGitDirectory(repoStorePath);
+        this.maxFileSize = maxFileSize;
         this.fsSizer = fsSizer;
     }
 
@@ -47,16 +68,25 @@ public class FSGitRepoStore implements RepoStore {
 
     @Override
     public ProjectRepo initRepo(String project) throws IOException {
-        GitProjectRepo ret = new GitProjectRepo(project);
+        GitProjectRepo ret = GitProjectRepo.fromName(project);
         ret.initRepo(this);
-        return ret;
+        return new WalkOverrideGitRepo(
+                ret, Optional.of(maxFileSize), Optional.empty());
     }
 
     @Override
     public ProjectRepo getExistingRepo(String project) throws IOException {
-        GitProjectRepo ret = new GitProjectRepo(project);
+        GitProjectRepo ret = GitProjectRepo.fromName(project);
         ret.useExistingRepository(this);
-        return ret;
+        return new WalkOverrideGitRepo(
+                ret, Optional.of(maxFileSize), Optional.empty());
+    }
+
+    @Override
+    public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
+        GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
+        return new WalkOverrideGitRepo(
+                ret, Optional.of(maxFileSize), Optional.of(commitId));
     }
 
     /* TODO: Perhaps we should just delete bad directories on the fly. */

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java

@@ -9,7 +9,7 @@ import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
 import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
-import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
 import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
 import uk.ac.ic.wlgitbridge.util.Log;
@@ -43,10 +43,19 @@ public class GitProjectRepo implements ProjectRepo {
     private final String projectName;
     private Optional<Repository> repository;
 
-    public GitProjectRepo(String projectName) {
+    public static GitProjectRepo fromJGitRepo(Repository repo) {
+        return new GitProjectRepo(
+                repo.getWorkTree().getName(), Optional.of(repo));
+    }
+
+    public static GitProjectRepo fromName(String projectName) {
+        return new GitProjectRepo(projectName, Optional.empty());
+    }
+
+    GitProjectRepo(String projectName, Optional<Repository> repository) {
         Preconditions.checkArgument(Project.isValidProjectName(projectName));
         this.projectName = projectName;
-        repository = Optional.empty();
+        this.repository = repository;
     }
 
     @Override
@@ -61,7 +70,10 @@ public class GitProjectRepo implements ProjectRepo {
         initRepositoryField(repoStore);
         Preconditions.checkState(repository.isPresent());
         Repository repo = this.repository.get();
-        Preconditions.checkState(!repo.getObjectDatabase().exists());
+        // TODO: assert that this is a fresh repo. At the moment, we can't be
+        // sure whether the repo to be init'd doesn't exist or is just fresh
+        // and we crashed / aborted while committing
+        if (repo.getObjectDatabase().exists()) return;
         repo.create();
     }
 
@@ -77,12 +89,12 @@ public class GitProjectRepo implements ProjectRepo {
     }
 
     @Override
-    public Map<String, RawFile> getFiles()
+    public RawDirectory getDirectory()
             throws IOException, GitUserException {
         Preconditions.checkState(repository.isPresent());
         return new RepositoryObjectTreeWalker(
                 repository.get()
-        ).getDirectoryContents().getFileTable();
+        ).getDirectoryContents(Optional.empty());
     }
 
     @Override

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java

@@ -2,13 +2,12 @@ package uk.ac.ic.wlgitbridge.bridge.repo;
 
 import org.eclipse.jgit.lib.Repository;
 import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
-import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
 import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Map;
 
 /**
  * Created by winston on 20/08/2016.
@@ -25,12 +24,12 @@ public interface ProjectRepo {
             RepoStore repoStore
     ) throws IOException;
 
-    Map<String, RawFile> getFiles(
+    RawDirectory getDirectory(
     ) throws IOException, GitUserException;
 
     Collection<String> commitAndGetMissing(
             GitDirectoryContents gitDirectoryContents
-    ) throws IOException;
+    ) throws IOException, GitUserException;
 
     void runGC() throws IOException;
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java

@@ -1,5 +1,8 @@
 package uk.ac.ic.wlgitbridge.bridge.repo;
 
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -21,6 +24,8 @@ public interface RepoStore {
 
     ProjectRepo getExistingRepo(String project) throws IOException;
 
+    ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
+
     void purgeNonexistentProjects(
             Collection<String> existingProjectNames
     );

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java

@@ -0,0 +1,21 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import javax.annotation.Nullable;
+import java.util.Optional;
+
+/**
+ * Created by winston on 02/07/2017.
+ */
+public class RepoStoreConfig {
+
+    @Nullable
+    private final Long maxFileSize;
+
+    public RepoStoreConfig(Long maxFileSize) {
+        this.maxFileSize = maxFileSize;
+    }
+
+    public Optional<Long> getMaxFileSize() {
+        return Optional.ofNullable(maxFileSize);
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java

@@ -0,0 +1,95 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Optional;
+
+/**
+ * This class takes a GitProjectRepo and delegates all calls to it.
+ *
+ * The purpose is to insert a file size check in {@link #getDirectory()}.
+ *
+ * We delegate instead of subclass because we can't override the static
+ * constructors in {@link GitProjectRepo}.
+ */
+public class WalkOverrideGitRepo implements ProjectRepo {
+
+    private final GitProjectRepo gitRepo;
+
+    private final Optional<Long> maxFileSize;
+
+    private final Optional<ObjectId> commitId;
+
+    public WalkOverrideGitRepo(
+            GitProjectRepo gitRepo,
+            Optional<Long> maxFileSize,
+            Optional<ObjectId> commitId
+    ) {
+        this.gitRepo = gitRepo;
+        this.maxFileSize = maxFileSize;
+        this.commitId = commitId;
+    }
+
+    @Override
+    public String getProjectName() {
+        return gitRepo.getProjectName();
+    }
+
+    @Override
+    public void initRepo(RepoStore repoStore) throws IOException {
+        gitRepo.initRepo(repoStore);
+    }
+
+    @Override
+    public void useExistingRepository(RepoStore repoStore) throws IOException {
+        gitRepo.useExistingRepository(repoStore);
+    }
+
+    @Override
+    public RawDirectory getDirectory() throws IOException, GitUserException {
+        Repository repo = gitRepo.getJGitRepository();
+        RepositoryObjectTreeWalker walker;
+        if (commitId.isPresent()) {
+            walker = new RepositoryObjectTreeWalker(repo, commitId.get());
+        } else {
+            walker = new RepositoryObjectTreeWalker(repo);
+        }
+        return walker.getDirectoryContents(maxFileSize);
+    }
+
+    @Override
+    public Collection<String> commitAndGetMissing(
+            GitDirectoryContents gitDirectoryContents
+    ) throws GitUserException, IOException {
+        return gitRepo.commitAndGetMissing(gitDirectoryContents);
+    }
+
+    @Override
+    public void runGC() throws IOException {
+        gitRepo.runGC();
+    }
+
+    @Override
+    public void deleteIncomingPacks() throws IOException {
+        gitRepo.deleteIncomingPacks();
+    }
+
+    @Override
+    public File getProjectDir() {
+        return gitRepo.getProjectDir();
+    }
+
+    @Override
+    public Repository getJGitRepository() {
+        return gitRepo.getJGitRepository();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java

@@ -1,9 +1,11 @@
 package uk.ac.ic.wlgitbridge.bridge.resource;
 
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Optional;
 
 /**
  * Created by winston on 20/08/2016.
@@ -15,7 +17,8 @@ public interface ResourceCache {
             String url,
             String newPath,
             Map<String, RawFile> fileTable,
-            Map<String, byte[]> fetchedUrls
-    ) throws IOException;
+            Map<String, byte[]> fetchedUrls,
+            Optional<Long> maxFileSize
+    ) throws IOException, SizeLimitExceededException;
 
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java

@@ -1,18 +1,19 @@
 package uk.ac.ic.wlgitbridge.bridge.resource;
 
-import com.ning.http.client.AsyncCompletionHandler;
-import com.ning.http.client.HttpResponseBodyPart;
-import com.ning.http.client.Response;
+import com.ning.http.client.*;
 import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
 import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
 import uk.ac.ic.wlgitbridge.snapshot.base.Request;
 import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
 import uk.ac.ic.wlgitbridge.util.Log;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 
 /**
@@ -32,13 +33,14 @@ public class UrlResourceCache implements ResourceCache {
             String url,
             String newPath,
             Map<String, RawFile> fileTable,
-            Map<String, byte[]> fetchedUrls
-    ) throws IOException {
+            Map<String, byte[]> fetchedUrls,
+            Optional<Long> maxFileSize
+    ) throws IOException, SizeLimitExceededException {
         String path = dbStore.getPathForURLInProject(projectName, url);
         byte[] contents;
         if (path == null) {
             path = newPath;
-            contents = fetch(projectName, url, path);
+            contents = fetch(projectName, url, path, maxFileSize);
             fetchedUrls.put(url, contents);
         } else {
             Log.info("Found (" + projectName + "): " + url);
@@ -54,7 +56,7 @@ public class UrlResourceCache implements ResourceCache {
                                     + "File url is: "
                                     + url
                     );
-                    contents = fetch(projectName, url, path);
+                    contents = fetch(projectName, url, path, maxFileSize);
                 } else {
                     contents = rawFile.getContents();
                 }
@@ -66,8 +68,9 @@ public class UrlResourceCache implements ResourceCache {
     private byte[] fetch(
             String projectName,
             final String url,
-            String path
-    ) throws FailedConnectionException {
+            String path,
+            Optional<Long> maxFileSize
+    ) throws FailedConnectionException, SizeLimitExceededException {
         byte[] contents;
         Log.info("GET -> " + url);
         try {
@@ -76,6 +79,28 @@ public class UrlResourceCache implements ResourceCache {
 
                 ByteArrayOutputStream bytes = new ByteArrayOutputStream();
 
+                @Override
+                public STATE onHeadersReceived(
+                        HttpResponseHeaders headers
+                ) throws SizeLimitExceededException {
+                    List<String> contentLengths
+                            = headers.getHeaders().get("Content-Length");
+                    if (!maxFileSize.isPresent()) {
+                        return STATE.CONTINUE;
+                    }
+                    if (contentLengths.isEmpty()) {
+                        return STATE.CONTINUE;
+                    }
+                    long contentLength = Long.parseLong(contentLengths.get(0));
+                    long maxFileSize_ = maxFileSize.get();
+                    if (contentLength <= maxFileSize_) {
+                        return STATE.CONTINUE;
+                    }
+                    throw new SizeLimitExceededException(
+                            Optional.of(path), contentLength, maxFileSize_
+                    );
+                }
+
                 @Override
                 public STATE onBodyPartReceived(
                         HttpResponseBodyPart bodyPart
@@ -115,6 +140,10 @@ public class UrlResourceCache implements ResourceCache {
             );
             throw new FailedConnectionException();
         } catch (ExecutionException e) {
+            Throwable cause = e.getCause();
+            if (cause instanceof SizeLimitExceededException) {
+                throw (SizeLimitExceededException) cause;
+            }
             Log.warn(
                     "ExecutionException when fetching project: " +
                             projectName +
@@ -126,6 +155,10 @@ public class UrlResourceCache implements ResourceCache {
             );
             throw new FailedConnectionException();
         }
+        if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
+            throw new SizeLimitExceededException(
+                    Optional.of(path), contents.length, maxFileSize.get());
+        }
         dbStore.addURLIndexForProject(projectName, url, path);
         return contents;
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java

@@ -25,6 +25,11 @@ public class ServletFile extends RawFile {
         return file.getContents();
     }
 
+    @Override
+    public long size() {
+        return getContents().length;
+    }
+
     public boolean isChanged() {
         return changed;
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java

@@ -1,6 +1,9 @@
 package uk.ac.ic.wlgitbridge.data.filestore;
 
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
+
 import java.util.Map;
+import java.util.Optional;
 
 /**
  * Created by Winston on 16/11/14.

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java

@@ -17,6 +17,8 @@ public abstract class RawFile {
 
     public abstract byte[] getContents();
 
+    public abstract long size();
+
     public final void writeToDisk(File directory) throws IOException {
         File file = new File(directory, getPath());
         file.getParentFile().mkdirs();

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java

@@ -23,4 +23,9 @@ public class RepositoryFile extends RawFile {
         return contents;
     }
 
+    @Override
+    public long size() {
+        return contents.length;
+    }
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java

@@ -5,6 +5,7 @@ import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.transport.ReceivePack;
 import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
 import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
 import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
 import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
 import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
@@ -28,9 +29,12 @@ import java.util.Optional;
 public class WLReceivePackFactory
         implements ReceivePackFactory<HttpServletRequest> {
 
+    private final RepoStore repoStore;
+
     private final Bridge bridge;
 
-    public WLReceivePackFactory(Bridge bridge) {
+    public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
+        this.repoStore = repoStore;
         this.bridge = bridge;
     }
 
@@ -69,7 +73,7 @@ public class WLReceivePackFactory
             hostname = httpServletRequest.getLocalName();
         }
         receivePack.setPreReceiveHook(
-                new WriteLatexPutHook(bridge, hostname, oauth2)
+                new WriteLatexPutHook(repoStore, bridge, hostname, oauth2)
         );
         return receivePack;
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java

@@ -7,12 +7,12 @@ import org.eclipse.jgit.transport.ReceiveCommand;
 import org.eclipse.jgit.transport.ReceiveCommand.Result;
 import org.eclipse.jgit.transport.ReceivePack;
 import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
 import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
 import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
 import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
 import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
-import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@@ -33,6 +33,8 @@ import java.util.Optional;
  */
 public class WriteLatexPutHook implements PreReceiveHook {
 
+    private final RepoStore repoStore;
+
     private final Bridge bridge;
     private final String hostname;
     private final Optional<Credential> oauth2;
@@ -41,15 +43,18 @@ public class WriteLatexPutHook implements PreReceiveHook {
      * The constructor to use, which provides the hook with the {@link Bridge},
      * the hostname (used to construct a URL to give to Overleaf to postback),
      * and the oauth2 (used to authenticate with the Snapshot API).
+     * @param repoStore
      * @param bridge the {@link Bridge}
      * @param hostname the hostname used for postback from the Snapshot API
      * @param oauth2 used to authenticate with the snapshot API, or null
      */
     public WriteLatexPutHook(
+            RepoStore repoStore,
             Bridge bridge,
             String hostname,
             Optional<Credential> oauth2
     ) {
+        this.repoStore = repoStore;
         this.bridge = bridge;
         this.hostname = hostname;
         this.oauth2 = oauth2;
@@ -152,18 +157,17 @@ public class WriteLatexPutHook implements PreReceiveHook {
             Repository repository,
             ReceiveCommand receiveCommand
     ) throws IOException, GitUserException {
-        return new RepositoryObjectTreeWalker(
-                repository,
-                receiveCommand.getNewId()
-        ).getDirectoryContents();
+        return repoStore
+                .useJGitRepo(repository, receiveCommand.getNewId())
+                .getDirectory();
     }
 
     private RawDirectory getOldDirectoryContents(
             Repository repository
     ) throws IOException, GitUserException {
-        return new RepositoryObjectTreeWalker(
-                repository
-        ).getDirectoryContents();
+        return repoStore
+                .useJGitRepo(repository, repository.resolve("HEAD"))
+                .getDirectory();
     }
 
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java

@@ -3,6 +3,7 @@ package uk.ac.ic.wlgitbridge.git.servlet;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jgit.http.server.GitServlet;
 import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
 import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
 import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
 import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
@@ -39,10 +40,11 @@ public class WLGitServlet extends GitServlet {
      */
     public WLGitServlet(
             ServletContextHandler ctxHandler,
+            RepoStore repoStore,
             Bridge bridge
     ) throws ServletException {
         setRepositoryResolver(new WLRepositoryResolver(bridge));
-        setReceivePackFactory(new WLReceivePackFactory(bridge));
+        setReceivePackFactory(new WLReceivePackFactory(repoStore, bridge));
         setUploadPackFactory(new WLUploadPackFactory());
         init(new WLGitServletConfig(ctxHandler));
     }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java

@@ -47,19 +47,7 @@ public class RepositoryObjectTreeWalker {
         this(repository, repository.resolve("HEAD~" + fromHead));
     }
 
-    public RawDirectory getDirectoryContents(
-    ) throws IOException, SizeLimitExceededException, InvalidGitRepository {
-        return getDirectoryContents(Optional.empty());
-    }
-
-    public RawDirectory getDirectoryContents(long maxFileSize)
-            throws InvalidGitRepository,
-                    SizeLimitExceededException,
-                    IOException {
-        return getDirectoryContents(Optional.of(maxFileSize));
-    }
-
-    private RawDirectory getDirectoryContents(Optional<Long> maxFileSize)
+    public RawDirectory getDirectoryContents(Optional<Long> maxFileSize)
             throws IOException,
                     SizeLimitExceededException,
                     InvalidGitRepository {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java

@@ -13,6 +13,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
 import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
 import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
 import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi;
 import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
 import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
@@ -52,7 +53,10 @@ public class GitBridgeServer {
         org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
         this.port = config.getPort();
         this.rootGitDirectoryPath = config.getRootGitDirectory();
-        RepoStore repoStore = new FSGitRepoStore(rootGitDirectoryPath);
+        RepoStore repoStore = new FSGitRepoStore(
+                rootGitDirectoryPath,
+                config.getRepoStore().flatMap(RepoStoreConfig::getMaxFileSize)
+        );
         DBStore dbStore = new SqliteDBStore(
                 Paths.get(
                         repoStore.getRootDirectory().getAbsolutePath()
@@ -61,14 +65,14 @@ public class GitBridgeServer {
         SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
         SnapshotApi snapshotApi = new NetSnapshotApi();
         bridge = Bridge.make(
+                config,
                 repoStore,
                 dbStore,
                 swapStore,
-                config.getSwapJob(),
                 snapshotApi
         );
         jettyServer = new Server(port);
-        configureJettyServer(config, snapshotApi);
+        configureJettyServer(config, repoStore, snapshotApi);
         SnapshotAPIRequest.setBasicAuth(
                 config.getUsername(),
                 config.getPassword()
@@ -110,11 +114,12 @@ public class GitBridgeServer {
 
     private void configureJettyServer(
             Config config,
+            RepoStore repoStore,
             SnapshotApi snapshotApi
     ) throws ServletException {
         HandlerCollection handlers = new HandlerList();
         handlers.addHandler(initApiHandler());
-        handlers.addHandler(initGitHandler(config, snapshotApi));
+        handlers.addHandler(initGitHandler(config, repoStore, snapshotApi));
         jettyServer.setHandler(handlers);
     }
 
@@ -136,6 +141,7 @@ public class GitBridgeServer {
 
     private Handler initGitHandler(
             Config config,
+            RepoStore repoStore,
             SnapshotApi snapshotApi
     ) throws ServletException {
         final ServletContextHandler servletContextHandler =
@@ -153,6 +159,7 @@ public class GitBridgeServer {
                 new ServletHolder(
                         new WLGitServlet(
                                 servletContextHandler,
+                                repoStore,
                                 bridge
                         )
                 ),

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getforversion/SnapshotFile.java

@@ -44,6 +44,11 @@ public class SnapshotFile extends RawFile implements JSONSource {
         return contents;
     }
 
+    @Override
+    public long size() {
+        return contents.length;
+    }
+
     /* Mock server */
 
     public SnapshotFile(String contents, String path) {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Instance.java

@@ -21,9 +21,11 @@ public class Instance {
 
     public static final JsonFactory jsonFactory = new GsonFactory();
 
-    public static final Gson prettyGson =
-            new GsonBuilder(
-            ).setPrettyPrinting().disableHtmlEscaping().create();
+    public static final Gson prettyGson = new GsonBuilder()
+            .setPrettyPrinting()
+            .serializeNulls()
+            .disableHtmlEscaping()
+            .create();
 
     public static final Gson gson = new Gson();
 

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java

@@ -95,7 +95,10 @@ public class ConfigTest {
                 "    \"oauth2ClientID\": \"<oauth2ClientID>\",\n" +
                 "    \"oauth2ClientSecret\": \"<oauth2ClientSecret>\",\n" +
                 "    \"oauth2Server\": \"https://www.overleaf.com\"\n" +
-                "  }\n" +
+                "  },\n" +
+                "  \"repoStore\": null,\n" +
+                "  \"swapStore\": null,\n" +
+                "  \"swapJob\": null\n" +
                 "}";
         assertEquals(
                 "sanitised config did not hide sensitive fields",

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java

@@ -2,6 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge;
 
 import org.junit.Before;
 import org.junit.Test;
+import uk.ac.ic.wlgitbridge.application.config.Config;
 import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
 import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
@@ -50,6 +51,18 @@ public class BridgeTest {
         swapJob = mock(SwapJob.class);
         gcJob = mock(GcJob.class);
         bridge = new Bridge(
+                new Config(
+                        0,
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                        "",
+                        null,
+                        null,
+                        null,
+                        null),
                 lock,
                 repoStore,
                 dbStore,

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java

@@ -10,6 +10,7 @@ import java.io.*;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Optional;
 
 import static org.junit.Assert.*;
 
@@ -42,7 +43,7 @@ public class FSGitRepoStoreTest {
         File tmp = makeTempRepoDir(tmpFolder, "rootdir");
         original = tmpFolder.newFolder("original");
         FileUtils.copyDirectory(tmp, original);
-        repoStore = new FSGitRepoStore(tmp.getAbsolutePath());
+        repoStore = new FSGitRepoStore(tmp.getAbsolutePath(), Optional.empty());
     }
 
     @Test

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java

@@ -56,7 +56,7 @@ public class GitProjectRepoTest {
     @Before
     public void setup() throws IOException {
         rootdir = makeTempRepoDir(tmpFolder, "rootdir");
-        repoStore = new FSGitRepoStore(rootdir.getAbsolutePath());
+        repoStore = new FSGitRepoStore(rootdir.getAbsolutePath(), Optional.empty());
         repo = fromExistingDir("repo");
         badGitignore = fromExistingDir("badgitignore");
         incoming = fromExistingDir("incoming");
@@ -64,7 +64,7 @@ public class GitProjectRepoTest {
     }
 
     private GitProjectRepo fromExistingDir(String dir) throws IOException {
-        GitProjectRepo ret = new GitProjectRepo(dir);
+        GitProjectRepo ret = GitProjectRepo.fromName(dir);
         ret.useExistingRepository(repoStore);
         return ret;
     }

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java

@@ -45,6 +45,7 @@ public class SwapJobImplTest {
                         tmpFolder,
                         "repostore"
                 ).getAbsolutePath(),
+                100_000,
                 FileUtils::sizeOfDirectory
         );
         dbStore = new SqliteDBStore(tmpFolder.newFile());

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java

@@ -9,7 +9,6 @@ import org.mockserver.client.server.MockServerClient;
 import org.mockserver.junit.MockServerRule;
 import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
-import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo;
 import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
 import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
 import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
@@ -20,6 +19,7 @@ import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockserver.model.HttpRequest.request;
@@ -67,12 +67,13 @@ public class ResourceFetcherTest {
         TemporaryFolder repositoryFolder = new TemporaryFolder();
         repositoryFolder.create();
         String repoStorePath = repositoryFolder.getRoot().getAbsolutePath();
-        RepoStore repoStore = new FSGitRepoStore(repoStorePath);
-        ProjectRepo repo = new GitProjectRepo("repo");
-        repo.initRepo(repoStore);
-        Map<String, RawFile> fileTable = repo.getFiles();
-        Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
-        resources.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
+        RepoStore repoStore = new FSGitRepoStore(repoStorePath, Optional.empty());
+        ProjectRepo repo = repoStore.initRepo("repo");
+        Map<String, RawFile> fileTable = repo.getDirectory().getFileTable();
+        Map<String, byte[]> fetchedUrls = new HashMap<>();
+        resources.get(
+                testProjectName, testUrl, newTestPath,
+                fileTable, fetchedUrls, Optional.empty());
 
         // We don't bother caching in this case, at present.
         assertEquals(0, fetchedUrls.size());