Merge pull request #1039 from sharelatex/ja-copybara

Sync to OSS repo using copybara

GitOrigin-RevId: 5ce0c08d614f3d84e7c3e1f9b3413681e5049103

services/web/Jenkinsfile

@@ -146,14 +146,23 @@ pipeline {
       }
     }
 
-
     stage('Sync OSS') {
       when {
         branch 'master'
       }
+      agent {
+        docker {
+          image 'sharelatex/copybara'
+          args "-u 0:0 -v /tmp/copybara:/root/copybara/cache"
+        }
+      }
       steps {
         sshagent (credentials: ['GIT_DEPLOY_KEY']) {
-          sh 'git push git@github.com:sharelatex/web-sharelatex.git HEAD:master'
+          sh 'git config --global user.name Copybot'
+          sh 'git config --global user.email copybot@overleaf.com'
+          sh 'mkdir -p /root/.ssh'
+          sh 'ssh-keyscan github.com >> /root/.ssh/known_hosts'
+          sh 'COPYBARA_CONFIG=./copybara/copy.bara.sky copybara --git-committer-email=copybot@overleaf.com --git-committer-name=Copybot || true'
         }
       }
     }

services/web/copybara/.gitignore

@@ -0,0 +1,2 @@
+.ssh/
+.cache/

services/web/copybara/README.md

@@ -0,0 +1,59 @@
+# Copybara Overleaf sync
+
+[Copybara](https://github.com/google/copybara) is a utility for syncing one
+git repository with another, while performing modifications, such as removing
+directories. We use this to keep a public OSS mirror of our web repo, but with
+the modules directory removed. The modules directory is where we place all of
+our proprietary code.
+
+## Running a sync locally
+
+You will need a copy of the `sharelatex/copybara` container, which can be pulled
+in, or built from the [copy.bara project](
+ https://github.com/google/copybara#getting-started-using-copybara):
+
+```bash
+> git clone git@github.com:google/copybara.git
+> cd copybara
+> docker build --rm -t sharelatex/copybara .
+```
+
+There is a `docker-compose.yml` file in this directory which configures
+everything. We mount out the copybara cache directory so we don't need to do a 
+full git clone each time.
+
+The `.ssh` directory in this directory should have the private key of the 
+`sharelatex-ci` GitHub account placed into it (can be retrieved from Jenkins),
+and have github.com pre-authorized:
+```bash
+> mkdir -p ./.ssh
+> ssh-keyscan github.com > ./.ssh/known_hosts
+> echo 'SHARELATEX_CI_PRIVATE_KEY' > ./.ssh/id_rsa
+```
+These are mounted into the container for use by copybara.
+
+## Initializing or fixing a bad state
+
+By default, copy.bara expects to find some metadata in the destination repo
+commits which it wrote on the last run. This tells it where to pick up syncing
+any new changes in the source repo. However, on the first run, or if things get
+in a bad state, you can provide with an explicit reference to a commit in the 
+source repo to start replaying commits from. Add the following to the
+`docker-compose.yml` config:
+```yaml
+  copybara:
+    ...
+    environment:
+      ...
+      COPYBARA_OPTIONS: "--last-rev=COMMIT_SHA_FROM_SOURCE_REPO"
+```
+
+If the destination repo gets out of sync in some way, reset its master branch
+to a point when things were in a good state, and then do a re-sync as above,
+but with the last-rev set to the corresponding good commit in the source repo.
+
+## Running a sync in CI
+
+The same `sharelatex/copybara` image and copybara config files is used by
+Jenkins to perform the sync at the end of a successful CI build of master. See
+the `Jenkinsfile` in the top level directory for this.

services/web/copybara/copy.bara.sky

@@ -0,0 +1,17 @@
+core.workflow(
+    name = "default",
+    origin = git.origin(
+      url = "git@github.com:sharelatex/web-sharelatex-internal.git",
+      ref = "master"
+    ),
+    destination = git.destination(
+      url = "git@github.com:sharelatex/web-sharelatex.git",
+      fetch = "master",
+      push = "master"
+    ),
+    # Exclude the modules directory
+    origin_files = glob(["**"], exclude = ["modules/**"]),
+    mode="ITERATIVE",
+    migrate_noop_changes=True,
+    authoring = authoring.pass_thru("Copybot <copybot@overleaf.com>")
+)

services/web/copybara/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '2'
+
+services:
+  copybara:
+    image: sharelatex/copybara
+    volumes:
+      - ./.ssh:/root/.ssh
+      - .:/usr/src/app
+      - ./.cache/:/root/copybara/cache/
+      - $HOME/.gitconfig:/root/.gitconfig
+    # Uncomment this to force copybara to start syncing from a certain commit
+    # environment:
+    #   COPYBARA_OPTIONS: "--last-rev=67edeed2c2d8c1d478c9a65d19020a301174cc8e"
+    command: copybara