Merge bug fixes with oauth2 skipping

services/git-bridge/bin/wlgb

@@ -56,7 +56,7 @@ do_start()
 	echo "Starting WriteLatex-Git Bridge..."
         start-stop-daemon --start --quiet --make-pidfile --pidfile $PIDFILE \
                 --exec /usr/bin/env LANG="C.UTF-8" $DAEMON -- \
-                $DAEMON_ARGS >> /var/log/wlgb/out.log 2>> /var/log/err.log &
+                $DAEMON_ARGS >> /var/log/wlgb/out.log 2>> /var/log/wlgb/err.log &
 	echo "WriteLatex-Git Bridge started."
 	echo "Config file at /etc/wlgb/config.json"
 	echo "Log file at /var/log/wlgb/out.log"

services/git-bridge/pom.xml

@@ -49,18 +49,14 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.10</version>
+            <version>4.12</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.jmock</groupId>
             <artifactId>jmock-junit4</artifactId>
             <version>2.6.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.jmock</groupId>
-            <artifactId>jmock-legacy</artifactId>
-            <version>2.6.0</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.eclipse.jetty.aggregate</groupId>
@@ -122,6 +118,12 @@
             <artifactId>commons-lang</artifactId>
             <version>2.6</version>
         </dependency>
+        <dependency>
+            <groupId>org.mock-server</groupId>
+            <artifactId>mockserver-netty</artifactId>
+            <version>3.10.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java

@@ -12,6 +12,7 @@ import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
 import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
 import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.data.model.db.SqlitePersistentStore;
 import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
 import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@@ -33,7 +34,7 @@ public class DataStore {
 
     public DataStore(String rootGitDirectoryPath) {
         rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath);
-        persistentStore = new PersistentStore(rootGitDirectory);
+        persistentStore = new SqlitePersistentStore(rootGitDirectory);
         List<String> excludedFromDeletion = persistentStore.getProjectNames();
         excludedFromDeletion.add(".wlgb");
         Util.deleteInDirectoryApartFrom(rootGitDirectory, excludedFromDeletion.toArray(new String[] {}));
@@ -44,10 +45,12 @@ public class DataStore {
 
     public void updateProjectWithName(Credential oauth2, String name, Repository repository) throws IOException, SnapshotPostException, GitAPIException, ForbiddenException {
         LinkedList<Snapshot> snapshots = snapshotFetcher.getSnapshotsForProjectAfterVersion(oauth2, name, persistentStore.getLatestVersionForProject(name));
+
+        makeCommitsFromSnapshots(name, repository, snapshots);
+
         if (!snapshots.isEmpty()) {
             persistentStore.setLatestVersionForProject(name, snapshots.getLast().getVersionID());
         }
-        makeCommitsFromSnapshots(name, repository, snapshots);
     }
 
     private void makeCommitsFromSnapshots(String name, Repository repository, List<Snapshot> snapshots) throws IOException, GitAPIException {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java

@@ -42,9 +42,15 @@ public class ResourceFetcher {
             if (contents == null) {
                 RawFile rawFile = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable().get(path);
                 if (rawFile == null) {
-                    throw new IllegalStateException("file was not in the current commit, or the git tree, yet path was not null");
+                    Util.sout(
+                        "WARNING: " +
+                        "File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
+                        "File url is: " + url
+                    );
+                    contents = fetch(projectName, url, path);
+                } else {
+                    contents = rawFile.getContents();
                 }
-                contents = rawFile.getContents();
             }
         }
         return new RepositoryFile(newPath, contents);
@@ -83,5 +89,4 @@ public class ResourceFetcher {
         persistentStore.addURLIndexForProject(projectName, url, path);
         return contents;
     }
-
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/PersistentStore.java

@@ -1,72 +1,20 @@
 package uk.ac.ic.wlgitbridge.data.model.db;
 
-import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
-
-import java.io.File;
-import java.sql.SQLException;
 import java.util.List;
 
 /**
- * Created by Winston on 19/11/14.
+ * Created by m on 20/11/15.
  */
-public class PersistentStore {
-
-    private final SQLiteWLDatabase database;
-
-    public PersistentStore(File rootGitDirectory) {
-        try {
-            database = new SQLiteWLDatabase(rootGitDirectory);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
-        }
-    }
+public interface PersistentStore {
+    List<String> getProjectNames();
 
-    public List<String> getProjectNames() {
-        try {
-            return database.getProjectNames();
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-    public void setLatestVersionForProject(String project, int versionID) {
-        try {
-            database.setVersionIDForProject(project, versionID);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
+    void setLatestVersionForProject(String project, int versionID);
 
-    public int getLatestVersionForProject(String project) {
-        try {
-            return database.getVersionIDForProjectName(project);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
+    int getLatestVersionForProject(String project);
 
-    public void addURLIndexForProject(String projectName, String url, String path) {
-        try {
-            database.addURLIndex(projectName, url, path);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
+    void addURLIndexForProject(String projectName, String url, String path);
 
-    public void deleteFilesForProject(String project, String... files) {
-        try {
-            database.deleteFilesForProject(project, files);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-    public String getPathForURLInProject(String projectName, String url) {
-        try {
-            return database.getPathForURLInProject(projectName, url);
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
+    void deleteFilesForProject(String project, String... files);
 
+    String getPathForURLInProject(String projectName, String url);
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java

@@ -0,0 +1,78 @@
+package uk.ac.ic.wlgitbridge.data.model.db;
+
+import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
+
+import java.io.File;
+import java.sql.SQLException;
+import java.util.List;
+
+/**
+ * Created by Winston on 19/11/14.
+ */
+public class SqlitePersistentStore implements PersistentStore {
+
+    private final SQLiteWLDatabase database;
+
+    public SqlitePersistentStore(File rootGitDirectory) {
+        try {
+            database = new SQLiteWLDatabase(rootGitDirectory);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public List<String> getProjectNames() {
+        try {
+            return database.getProjectNames();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    @Override
+    public void setLatestVersionForProject(String project, int versionID) {
+        try {
+            database.setVersionIDForProject(project, versionID);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public int getLatestVersionForProject(String project) {
+        try {
+            return database.getVersionIDForProjectName(project);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void addURLIndexForProject(String projectName, String url, String path) {
+        try {
+            database.addURLIndex(projectName, url, path);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void deleteFilesForProject(String project, String... files) {
+        try {
+            database.deleteFilesForProject(project, files);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    @Override
+    public String getPathForURLInProject(String projectName, String url) {
+        try {
+            return database.getPathForURLInProject(projectName, url);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java

@@ -67,6 +67,7 @@ public class GetDocResult extends Result {
         JsonObject jsonObject = json.getAsJsonObject();
         if (jsonObject.has("status")) {
             switch (jsonObject.get("status").getAsInt()) {
+            case 401:
             case 403:
                 forbidden = new ForbiddenException();
                 break;

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java

@@ -0,0 +1,74 @@
+package uk.ac.ic.wlgitbridge.data.model;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.mockserver.model.HttpRequest.request;
+import static org.mockserver.model.HttpResponse.response;
+
+import org.eclipse.jgit.lib.*;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockserver.client.server.MockServerClient;
+import org.mockserver.junit.MockServerRule;
+import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
+
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by m on 20/11/15.
+ */
+public class ResourceFetcherTest {
+    @Rule
+    public MockServerRule mockServerRule = new MockServerRule(this);
+
+    private MockServerClient mockServerClient;
+
+    @Test
+    public void fetchesFilesThatAreMissingFromUrlStoreCache() throws IOException {
+        final String testProjectName = "123abc";
+        final String testUrl = "http://localhost:" + mockServerRule.getHttpPort() + "/123abc";
+        final String oldTestPath = "testPath";
+        final String newTestPath = "missingPath";
+
+        mockServerClient.when(
+            request()
+            .withMethod("GET")
+            .withPath("/123abc")
+        )
+        .respond(
+            response()
+            .withStatusCode(200)
+            .withBody("content")
+        );
+
+        final Mockery context = new Mockery();
+        final PersistentStore persistentStore = context.mock(PersistentStore.class);
+        context.checking(new Expectations() {{
+            // It should fetch the file once it finds it is missing.
+            oneOf(persistentStore).getPathForURLInProject(testProjectName, testUrl);
+            will(returnValue(oldTestPath));
+
+            // It should update the URL index store once it has fetched; at present, it does not actually change the stored path.
+            oneOf(persistentStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
+        }});
+
+        ResourceFetcher resourceFetcher = new ResourceFetcher(persistentStore);
+        TemporaryFolder repositoryFolder = new TemporaryFolder();
+        repositoryFolder.create();
+        Repository repository = new FileRepositoryBuilder().setWorkTree(repositoryFolder.getRoot()).build();
+        Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
+        resourceFetcher.get(testProjectName, testUrl, newTestPath, repository, fetchedUrls);
+
+        // We don't bother caching in this case, at present.
+        assertEquals(0, fetchedUrls.size());
+    }
+}