Merge pull request #45 from overleaf/sk-migrate-repos-v1-v2

Migrate repos from v1 to v2

services/git-bridge/pom.xml

@@ -9,6 +9,7 @@
     <version>1.0-SNAPSHOT</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <!--<maven.test.skip>true</maven.test.skip>-->
     </properties>
     <build>
         <plugins>
@@ -22,6 +23,15 @@
                     <compilerArgument></compilerArgument>
                 </configuration>
             </plugin>
+            <!-- Workaround, test loader crashes without this configuration option -->
+            <!-- See: https://stackoverflow.com/questions/53010200/maven-surefire-could-not-find-forkedbooter-class -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                  <argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
+                </configuration>
+            </plugin>
             <!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-assembly-plugin -->
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java

@@ -37,6 +37,7 @@ import uk.ac.ic.wlgitbridge.server.PostbackContents;
 import uk.ac.ic.wlgitbridge.server.PostbackHandler;
 import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
 import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
 import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
 import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
 import uk.ac.ic.wlgitbridge.snapshot.push.PostbackPromise;
@@ -299,7 +300,7 @@ public class Bridge {
      * Synchronises the given repository with Overleaf.
      *
      * It acquires the project lock and calls
-     * {@link #getUpdatedRepoCritical(Optional, String)}.
+     * {@link #getUpdatedRepoCritical(Optional, String, GetDocResult)}.
      * @param oauth2 The oauth2 to use
      * @param projectName The name of the project
      * @throws IOException
@@ -310,11 +311,13 @@ public class Bridge {
             String projectName
     ) throws IOException, GitUserException {
         try (LockGuard __ = lock.lockGuard(projectName)) {
-            if (!snapshotAPI.projectExists(oauth2, projectName)) {
+            Optional<GetDocResult> maybeDoc = snapshotAPI.getDoc(oauth2, projectName);
+            if (!maybeDoc.isPresent()) {
                 throw new RepositoryNotFoundException(projectName);
             }
+            GetDocResult doc = maybeDoc.get();
             Log.info("[{}] Updating repository", projectName);
-            return getUpdatedRepoCritical(oauth2, projectName);
+            return getUpdatedRepoCritical(oauth2, projectName, doc);
         }
     }
 
@@ -346,14 +349,51 @@ public class Bridge {
      */
     private ProjectRepo getUpdatedRepoCritical(
             Optional<Credential> oauth2,
-            String projectName
+            String projectName,
+            GetDocResult doc
     ) throws IOException, GitUserException {
         ProjectRepo repo;
         ProjectState state = dbStore.getProjectState(projectName);
         switch (state) {
         case NOT_PRESENT:
-            repo = repoStore.initRepo(projectName);
-            break;
+            Log.info("[{}] Repo not present", projectName);
+            String migratedFromID = doc.getMigratedFromID();
+            if (migratedFromID != null) {
+                Log.info("[{}] Has a migratedFromId: {}", projectName, migratedFromID);
+                try (LockGuard __ = lock.lockGuard(migratedFromID)) {
+                    ProjectState sourceState = dbStore.getProjectState(migratedFromID);
+                    switch (sourceState) {
+                        case NOT_PRESENT:
+                            // Normal init-repo
+                            Log.info("[{}] migrated-from project not present, proceed as normal",
+                                 projectName
+                            );
+                            repo = repoStore.initRepo(projectName);
+                            break;
+                        case SWAPPED:
+                            // Swap back and then copy
+                            swapJob.restore(migratedFromID);
+                            /* Fallthrough */
+                        default:
+                            // Copy data, and set version to zero
+                            Log.info("[{}] Init from other project: {}",
+                                projectName,
+                                migratedFromID
+                            );
+                            repo = repoStore.initRepoFromExisting(projectName, migratedFromID);
+                            dbStore.setLatestVersionForProject(migratedFromID, 0);
+                            dbStore.setLastAccessedTime(
+                                    migratedFromID,
+                                    Timestamp.valueOf(LocalDateTime.now())
+
+                            );
+                    }
+                }
+                break;
+            } else {
+                repo = repoStore.initRepo(projectName);
+                break;
+            }
         case SWAPPED:
             swapJob.restore(projectName);
             /* Fallthrough */

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java

@@ -4,6 +4,7 @@ import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import org.apache.commons.io.FileUtils;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Repository;
+import uk.ac.ic.wlgitbridge.util.Log;
 import uk.ac.ic.wlgitbridge.util.Project;
 import uk.ac.ic.wlgitbridge.util.Tar;
 
@@ -74,6 +75,27 @@ public class FSGitRepoStore implements RepoStore {
                 ret, Optional.of(maxFileSize), Optional.empty());
     }
 
+     @Override
+     public ProjectRepo initRepoFromExisting(
+             String project, String fromProject
+     ) throws IOException {
+         String repoRoot = getRepoStorePath();
+         String sourcePath = repoRoot + "/" + fromProject;
+         String destinationPath = repoRoot + "/" + project;
+         Log.info("[{}] Init repo by copying data from: {}, to: {}",
+             project,
+             sourcePath,
+             destinationPath
+         );
+         File source = new File(sourcePath);
+         File destination = new File(destinationPath);
+         FileUtils.copyDirectory(source, destination);
+         GitProjectRepo ret = GitProjectRepo.fromName(project);
+         ret.useExistingRepository(this);
+         return new WalkOverrideGitRepo(
+                 ret, Optional.of(maxFileSize), Optional.empty());
+     }
+
     @Override
     public ProjectRepo getExistingRepo(String project) throws IOException {
         GitProjectRepo ret = GitProjectRepo.fromName(project);

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java

@@ -22,6 +22,8 @@ public interface RepoStore {
 
     ProjectRepo initRepo(String project) throws IOException;
 
+    ProjectRepo initRepoFromExisting(String project, String fromProject) throws IOException;
+
     ProjectRepo getExistingRepo(String project) throws IOException;
 
     ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java

@@ -44,6 +44,20 @@ public class SnapshotApiFacade {
         }
     }
 
+    public Optional<GetDocResult> getDoc(
+            Optional<Credential> oauth2,
+            String projectName
+    ) throws FailedConnectionException, GitUserException {
+        try {
+            GetDocResult doc = SnapshotApi
+                    .getResult(api.getDoc(oauth2, projectName));
+            doc.getVersionID();
+            return Optional.of(doc);
+        } catch (InvalidProjectException e) {
+            return Optional.empty();
+        }
+    }
+
     public Deque<Snapshot> getSnapshots(
             Optional<Credential> oauth2,
             String projectName,

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/MissingRepositoryException.java

@@ -15,11 +15,31 @@ public class MissingRepositoryException extends SnapshotAPIException {
         "If this problem persists, please contact us."
     );
 
-    public static final List<String> EXPORTED_TO_V2 = Arrays.asList(
-        "This Overleaf project has been moved to Overleaf v2, and git access is temporarily unsupported.",
-        "",
-        "See https://www.overleaf.com/help/342 for more information."
-    );
+    static List<String> buildExportedToV2Message(String remoteUrl) {
+        if (remoteUrl == null) {
+            return Arrays.asList(
+                    "This Overleaf project has been moved to Overleaf v2 and cannot be used with git at this time.",
+                    "",
+                    "If this error persists, please contact us at support@overleaf.com, or",
+                    "see https://www.overleaf.com/help/342 for more information."
+            );
+        } else {
+            return Arrays.asList(
+                    "This Overleaf project has been moved to Overleaf v2 and has a new identifier.",
+                    "Please update your remote to:",
+                    "",
+                    "    " + remoteUrl,
+                    "",
+                    "Assuming you are using the default \"origin\" remote, the following commands",
+                    "will change the remote for you:",
+                    "",
+                    "    git remote set-url origin " + remoteUrl,
+                    "",
+                    "If this does not work, please contact us at support@overleaf.com, or",
+                    "see https://www.overleaf.com/help/342 for more information."
+            );
+        }
+    }
 
     private List<String> descriptionLines;
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/Request.java

@@ -79,9 +79,17 @@ public abstract class Request<T extends Result> {
                     try {
                         JsonObject json = Instance.gson.fromJson(httpCause.getContent(), JsonObject.class);
                         String message = json.get("message").getAsString();
+                        String newRemote;
+                        if (json.has("newRemote")) {
+                            newRemote = json.get("newRemote").getAsString();
+                        } else {
+                            newRemote = null;
+                        }
 
                         if ("Exported to v2".equals(message)) {
-                            throw new MissingRepositoryException(MissingRepositoryException.EXPORTED_TO_V2);
+                            throw new MissingRepositoryException(
+                                MissingRepositoryException.buildExportedToV2Message(newRemote)
+                            );
                         }
                     } catch (IllegalStateException
                             | ClassCastException

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getdoc/GetDocResult.java

@@ -19,6 +19,7 @@ public class GetDocResult extends Result {
 
     private int error;
     private int versionID;
+    private String migratedFromID;
     private String createdAt;
     private WLUser user;
 
@@ -37,7 +38,8 @@ public class GetDocResult extends Result {
             int versionID,
             String createdAt,
             String email,
-            String name
+            String name,
+            String migratedFromID
     ) {
         if (error == null) {
             this.error = -1;
@@ -47,6 +49,7 @@ public class GetDocResult extends Result {
         this.versionID = versionID;
         this.createdAt = createdAt;
         this.user = new WLUser(name, email);
+        this.migratedFromID = migratedFromID;
     }
 
     @Override
@@ -59,6 +62,9 @@ public class GetDocResult extends Result {
             latestVerBy.addProperty("email", getEmail());
             latestVerBy.addProperty("name", getName());
             jsonThis.add("latestVerBy", latestVerBy);
+            if (migratedFromID != null) {
+                jsonThis.addProperty("migratedFromId", migratedFromID);
+            }
         } else {
             jsonThis.addProperty("status", error);
             String message;
@@ -93,6 +99,11 @@ public class GetDocResult extends Result {
         } else {
             versionID = jsonObject.get("latestVerId").getAsInt();
             createdAt = jsonObject.get("latestVerAt").getAsString();
+            if (jsonObject.has("migratedFromId")) {
+                migratedFromID = jsonObject.get("migratedFromId").getAsString();
+            } else {
+                migratedFromID = null;
+            }
             String name = null;
             String email = null;
             JsonElement latestVerBy = jsonObject.get("latestVerBy");
@@ -126,4 +137,6 @@ public class GetDocResult extends Result {
         return user.getEmail();
     }
 
+    public String getMigratedFromID() { return migratedFromID; }
+
 }

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/servermock/state/SnapshotAPIState.java

@@ -48,7 +48,8 @@ public class SnapshotAPIState {
                         243,
                         "2014-11-30T18:40:58Z",
                         "jdleesmiller+1@gmail.com",
-                        "John+1"
+                        "John+1",
+                        null
                 )
         );
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/servermock/state/SnapshotAPIStateBuilder.java

@@ -83,6 +83,10 @@ public class SnapshotAPIStateBuilder {
             String projectName,
             JsonObject jsonGetDoc
     ) {
+        String migratedFromId = null;
+        if (jsonGetDoc.has("migratedFromId")) {
+            migratedFromId = jsonGetDoc.get("migratedFromId").getAsString();
+        }
         getDoc.put(
                 projectName,
                 new GetDocResult(
@@ -90,7 +94,8 @@ public class SnapshotAPIStateBuilder {
                         jsonGetDoc.get("versionID").getAsInt(),
                         jsonGetDoc.get("createdAt").getAsString(),
                         jsonGetDoc.get("email").getAsString(),
-                        jsonGetDoc.get("name").getAsString()
+                        jsonGetDoc.get("name").getAsString(),
+                        migratedFromId
                 )
         );
     }

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java

@@ -117,6 +117,12 @@ public class WLGitBridgeIntegrationTest {
         put("pushSubmoduleFailsWithInvalidGitRepo", new HashMap<String, SnapshotAPIState>() {{
             put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/pushSubmoduleFailsWithInvalidGitRepo/state/state.json")).build());
         }});
+        put("canMigrateRepository", new HashMap<String, SnapshotAPIState>() {{
+            put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/canMigrateRepository/state/state.json")).build());
+        }});
+        put("skipMigrationWhenMigratedFromMissing", new HashMap<String, SnapshotAPIState>() {{
+            put("state", new SnapshotAPIStateBuilder(getResourceAsStream("/skipMigrationWhenMigratedFromMissing/state/state.json")).build());
+        }});
     }};
 
     @Rule
@@ -787,6 +793,43 @@ public class WLGitBridgeIntegrationTest {
         assertNotEquals(0, gitProcess.waitFor());
     }
 
+    @Test
+    public void canMigrateRepository() throws IOException, GitAPIException, InterruptedException {
+        int gitBridgePort = 33881;
+        int mockServerPort = 3881;
+        MockSnapshotServer server = new MockSnapshotServer(mockServerPort, getResource("/canMigrateRepository").toFile());
+        server.start();
+        server.setState(states.get("canMigrateRepository").get("state"));
+        GitBridgeApp wlgb = new GitBridgeApp(new String[] {
+                makeConfigFile(gitBridgePort, mockServerPort)
+        });
+        wlgb.run();
+        File testprojDir = gitClone("testproj", gitBridgePort, dir);
+        File testprojDir2 = gitClone("testproj2", gitBridgePort, dir);
+        wlgb.stop();
+
+        // Second project content is equal to content of the first
+        assertTrue(FileUtil.gitDirectoriesAreEqual(getResource("/canMigrateRepository/state/testproj"), testprojDir2.toPath()));
+    }
+
+    @Test
+    public void skipMigrationWhenMigratedFromMissing() throws IOException, GitAPIException, InterruptedException {
+        int gitBridgePort = 33882;
+        int mockServerPort = 3882;
+        MockSnapshotServer server = new MockSnapshotServer(mockServerPort, getResource("/skipMigrationWhenMigratedFromMissing").toFile());
+        server.start();
+        server.setState(states.get("skipMigrationWhenMigratedFromMissing").get("state"));
+        GitBridgeApp wlgb = new GitBridgeApp(new String[] {
+                makeConfigFile(gitBridgePort, mockServerPort)
+        });
+        wlgb.run();
+        // don't clone the source project first
+        File testprojDir2 = gitClone("testproj2", gitBridgePort, dir);
+        wlgb.stop();
+
+        assertTrue(FileUtil.gitDirectoriesAreEqual(getResource("/skipMigrationWhenMigratedFromMissing/state/testproj2"), testprojDir2.toPath()));
+    }
+
     private String makeConfigFile(
             int port,
             int apiPort

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java

@@ -14,6 +14,7 @@ import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
 import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
 import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
 import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
 
 import java.io.IOException;
 import java.util.ArrayDeque;
@@ -91,6 +92,9 @@ public class BridgeTest {
         when(repoStore.getExistingRepo("asdf")).thenReturn(repo);
         when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT);
         when(snapshotAPI.projectExists(Optional.empty(), "asdf")).thenReturn(true);
+        when(
+            snapshotAPI.getDoc(Optional.empty(), "asdf")
+        ).thenReturn(Optional.of(mock(GetDocResult.class)));
         when(
                 snapshotAPI.getSnapshots(
                         any(),

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/state.json

@@ -0,0 +1,83 @@
+[
+  {
+    "project": "testproj2",
+    "getDoc": {
+      "versionID": 1,
+      "createdAt": "2014-11-30T18:40:58.123Z",
+      "email": "jdleesmiller+1@gmail.com",
+      "name": "John+1",
+      "migratedFromId": "testproj"
+    },
+    "getSavedVers": [],
+    "getForVers": [
+      {
+        "versionID": 1,
+        "srcs": [
+          {
+            "content": "content\n",
+            "path": "main.tex"
+          },
+          {
+            "content": "This text is from another file.",
+            "path": "foo/bar/test.tex"
+          }
+        ],
+        "atts": [
+          {
+            "url": "http://127.0.0.1:3857/state/testproj/min_mean_wait_evm_7_eps_150dpi.png",
+            "path": "min_mean_wait_evm_7_eps_150dpi.png"
+          }
+        ]
+      }
+    ],
+    "push": "success",
+    "postback": {
+      "type": "success",
+      "versionID": 2
+    }
+  },
+  {
+    "project": "testproj",
+    "getDoc": {
+      "versionID": 1,
+      "createdAt": "2014-11-30T18:40:58.123Z",
+      "email": "jdleesmiller+1@gmail.com",
+      "name": "John+1"
+    },
+    "getSavedVers": [
+      {
+        "versionID": 1,
+        "comment": "added more info on doc GET and error details",
+        "email": "jdleesmiller+1@gmail.com",
+        "name": "John+1",
+        "createdAt": "2014-11-30T18:47:01.333Z"
+      }
+    ],
+    "getForVers": [
+      {
+        "versionID": 1,
+        "srcs": [
+          {
+            "content": "content\n",
+            "path": "main.tex"
+          },
+          {
+            "content": "This text is from another file.",
+            "path": "foo/bar/test.tex"
+          }
+        ],
+        "atts": [
+          {
+            "url": "http://127.0.0.1:3857/state/testproj/min_mean_wait_evm_7_eps_150dpi.png",
+            "path": "min_mean_wait_evm_7_eps_150dpi.png"
+          }
+        ]
+      }
+    ],
+    "push": "success",
+    "postback": {
+      "type": "success",
+      "versionID": 2
+    }
+  }
+]

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/testproj/foo/bar/test.tex

@@ -0,0 +1 @@
+This text is from another file.

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/canMigrateRepository/state/testproj/main.tex

@@ -0,0 +1 @@
+content

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/state.json

@@ -0,0 +1,39 @@
+[
+  {
+    "project": "testproj2",
+    "getDoc": {
+      "versionID": 1,
+      "createdAt": "2014-11-30T18:40:58.123Z",
+      "email": "jdleesmiller+1@gmail.com",
+      "name": "John+1",
+      "migratedFromId": "testprojthatdoesnotexist"
+    },
+    "getSavedVers": [],
+    "getForVers": [
+      {
+        "versionID": 1,
+        "srcs": [
+          {
+            "content": "two\n",
+            "path": "main.tex"
+          },
+          {
+            "content": "This text is from another file.",
+            "path": "foo/bar/test.tex"
+          }
+        ],
+        "atts": [
+          {
+            "url": "http://127.0.0.1:3857/state/testproj/min_mean_wait_evm_7_eps_150dpi.png",
+            "path": "min_mean_wait_evm_7_eps_150dpi.png"
+          }
+        ]
+      }
+    ],
+    "push": "success",
+    "postback": {
+      "type": "success",
+      "versionID": 2
+    }
+  }
+]

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/testproj2/foo/bar/test.tex

@@ -0,0 +1 @@
+This text is from another file.

services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/WLGitBridgeIntegrationTest/skipMigrationWhenMigratedFromMissing/state/testproj2/main.tex

@@ -0,0 +1 @@
+two