Fix issue with cloning multiple identical files and add test

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java

@@ -4,12 +4,12 @@ import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.Repository;
-import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
 import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
 import uk.ac.ic.wlgitbridge.data.SnapshotFetcher;
 import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
-import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
 import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
 import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
 import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
 import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@@ -17,10 +17,7 @@ import uk.ac.ic.wlgitbridge.util.Util;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.TimeZone;
+import java.util.*;
 
 /**
  * Created by Winston on 06/11/14.
@@ -55,8 +52,9 @@ public class DataStore {
         for (Snapshot snapshot : snapshots) {
             List<RawFile> files = new LinkedList<RawFile>();
             files.addAll(snapshot.getSrcs());
+            Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
             for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
-                files.add(resourceFetcher.get(name, snapshotAttachment.getUrl(), snapshotAttachment.getPath(), repository));
+                files.add(resourceFetcher.get(name, snapshotAttachment.getUrl(), snapshotAttachment.getPath(), repository, fetchedUrls));
             }
             commit(name, new GitDirectoryContents(files, rootGitDirectory, name, snapshot), repository);
         }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java

@@ -14,6 +14,7 @@ import uk.ac.ic.wlgitbridge.util.Util;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 /**
@@ -27,16 +28,22 @@ public class ResourceFetcher {
         this.persistentStore = persistentStore;
     }
 
-    public RawFile get(String projectName, String url, String newPath, Repository repository) throws IOException {
+    public RawFile get(String projectName, String url, String newPath, Repository repository, Map<String, byte[]> fetchedUrls) throws IOException {
         String path = persistentStore.getPathForURLInProject(projectName, url);
         byte[] contents;
         if (path == null) {
             path = newPath;
             contents = fetch(projectName, url, path);
+            fetchedUrls.put(url, contents);
         } else {
             Util.sout("Found (" + projectName + "): " + url);
             Util.sout("At (" + projectName + "): " + path);
-            contents = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable().get(path).getContents();
+            RawFile rawFile = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable().get(path);
+            if (rawFile != null) {
+                contents = rawFile.getContents();
+            } else {
+                contents = fetchedUrls.get(url);
+            }
         }
         return new RepositoryFile(newPath, contents);
     }

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest.java

@@ -60,6 +60,9 @@ public class WLGitBridgeIntegrationTest {
                     put("base", new SnapshotAPIStateBuilder(getResourceAsStream("/canPullADuplicateBinaryFile/base/state.json")).build());
                     put("withDuplicateBinaryFile", new SnapshotAPIStateBuilder(getResourceAsStream("/canPullADuplicateBinaryFile/withDuplicateBinaryFile/state.json")).build());
                 }});
+                put("canCloneDuplicateBinaryFiles", new HashMap<String, SnapshotAPIState>() {{
+                    put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canCloneDuplicateBinaryFiles/state/state.json")).build());
+                }});
                 put("canPullAModifiedNestedFile", new HashMap<String, SnapshotAPIState>() {{
                     put("base", new SnapshotAPIStateBuilder(getResourceAsStream("/canPullAModifiedNestedFile/base/state.json")).build());
                     put("withModifiedNestedFile", new SnapshotAPIStateBuilder(getResourceAsStream("/canPullAModifiedNestedFile/withModifiedNestedFile/state.json")).build());
@@ -273,6 +276,24 @@ public class WLGitBridgeIntegrationTest {
         assertTrue(FileUtil.gitDirectoriesAreEqual(getResource("/canPullADuplicateBinaryFile/withDuplicateBinaryFile/testproj"), testprojDir.toPath()));
     }
 
+    @Test
+    public void canCloneDuplicateBinaryFiles() throws IOException, GitAPIException, InterruptedException {
+        MockSnapshotServer server = new MockSnapshotServer(4002, getResource("/canCloneDuplicateBinaryFiles").toFile());
+        server.start();
+        server.setState(states.get("canCloneDuplicateBinaryFiles").get("state"));
+        GitBridgeApp wlgb = new GitBridgeApp(new String[] {
+                makeConfigFile(44002, 4002)
+        });
+        wlgb.run();
+        File dir = folder.newFolder();
+        Process git = runtime.exec("git clone http://127.0.0.1:44002/testproj.git", null, dir);
+        int exitCode = git.waitFor();
+        wlgb.stop();
+        File testprojDir = new File(dir, "testproj");
+        assertEquals(0, exitCode);
+        assertTrue(FileUtil.gitDirectoriesAreEqual(getResource("/canCloneDuplicateBinaryFiles/state/testproj"), testprojDir.toPath()));
+    }
+
     @Test
     public void canPullAModifiedNestedFile() throws IOException, GitAPIException, InterruptedException {
         MockSnapshotServer server = new MockSnapshotServer(3864, getResource("/canPullAModifiedNestedFile").toFile());

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canCloneDuplicateBinaryFiles/state/state.json

@@ -0,0 +1,50 @@
+[
+	{
+		"project": "testproj",
+		"getDoc": {
+			"versionID": 1,
+			"createdAt": "2014-11-30T18:40:58.123Z",
+			"email": "jdleesmiller+1@gmail.com",
+			"name": "John+1"
+		},
+		"getSavedVers": [
+        	{
+        		"versionID": 1,
+        		"comment": "added more info on doc GET and error details",
+        		"email": "jdleesmiller+1@gmail.com",
+        		"name": "John+1",
+        		"createdAt": "2014-11-30T18:47:01.456Z"
+        	}
+		],
+		"getForVers": [
+			{
+				"versionID": 1,
+				"srcs": [
+                    {
+                    	"content": "content\n",
+                    	"path": "main.tex"
+                    },
+                    {
+                    	"content": "This text is from another file.",
+                    	"path": "test.tex"
+                    }
+	            ],
+	            "atts": [
+	            	{
+	            		"url": "http://127.0.0.1:4002/state/testproj/overleaf-white-410.png",
+	            		"path": "overleaf-white-410-copy.png"
+	            	},
+					{
+						"url": "http://127.0.0.1:4002/state/testproj/overleaf-white-410.png",
+						"path": "overleaf-white-410.png"
+					}
+	            ]
+	        }
+		],
+		"push": "success",
+		"postback": {
+			"type": "success",
+			"versionID": 2
+		}
+	}
+]

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canCloneDuplicateBinaryFiles/state/testproj/main.tex

@@ -0,0 +1 @@
+content

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canCloneDuplicateBinaryFiles/state/testproj/test.tex

@@ -0,0 +1 @@
+This text is from another file.