Merge pull request #9725 from overleaf/ae-overleaf-develop-ce

Add Server CE development environment

GitOrigin-RevId: c91d9eb58b1dbc3be24646e77033cd3242e707f0

develop/.gitignore

@@ -0,0 +1,3 @@
+/compiles/*
+!.gitkeep
+.env

develop/README.md

@@ -0,0 +1,79 @@
+# Overleaf Community Edition, development environment
+
+## Building and running
+
+In this `develop` directory, build and start the services:
+
+```shell
+bin/up
+```
+
+Once the services are running, open <http://localhost/launchpad> to create the first admin account.
+
+After making any changes to the code, run `bin/up` manually to rebuild the changed Docker images and recreate the changed containers.
+
+## TeX Live
+
+Compiling a PDF requires building a TeX Live image to handle the compilation inside Docker:
+
+```shell
+docker build texlive -t texlive-full
+```
+
+> **Note**
+> To compile on a macOS host, you'll need to override the path to the Docker socket by creating a `.env` file in this directory, containing
+> `DOCKER_SOCKET_PATH=/var/run/docker.sock.raw`
+
+## Development
+
+To avoid running `bin/up` after every code change, you can run Overleaf
+Community Edition in _development mode_, where services will automatically
+update on code changes. To do this, use the included `bin/dev` script:
+
+```shell
+bin/dev
+```
+
+This will start all services using `nodemon`, which will automatically monitor
+the code and restart the services as necessary. This will incur a performance
+hit, especially on macOS, so in order to only start a subset of the services,
+provide a space-separated list to the `bin/dev` script.
+
+```shell
+bin/dev [service1] [service2] ...  [serviceN]
+```
+
+> **Note**
+> Starting the `web` service in _development mode_ will only update the `web`
+> service when backend code changes. In order to automatically update frontend
+> code as well, make sure to start the `webpack` service in _development mode_
+> as well.
+
+## Debugging
+
+When run in _development mode_ most services expose a debugging port to which
+you can attach a debugger such as
+[the inspector in Chrome's Dev Tools](chrome://inspect/) or one integrated into
+an IDE. The following table shows the port exposed on the **host machine** for
+each service:
+
+| Service            | Port |
+| ------------------ | ---- |
+| `web`              | 9229 |
+| `clsi`             | 9230 |
+| `chat`             | 9231 |
+| `contacts`         | 9232 |
+| `docstore`         | 9233 |
+| `document-updater` | 9234 |
+| `filestore`        | 9235 |
+| `notifications`    | 9236 |
+| `real-time`        | 9237 |
+| `track-changes`    | 9238 |
+
+To attach to a service using Chrome's _remote debugging_, go to
+<chrome://inspect/> and make sure _Discover network targets_ is checked. Next
+click _Configure..._ and add an entry `localhost:[service port]` for each of the
+services you want to attach a debugger to.
+
+After adding an entry, the service will show up as a _Remote Target_ that you
+can inspect and debug.

develop/bin/dev

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --no-deps --detach "$@"

develop/bin/down

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose down

develop/bin/logs

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+docker-compose logs --follow --tail 10 --no-color "$@" \
+  | ggrep --line-buffered --invert-match "global.gc" \
+  | ggrep --line-buffered --invert-match "health.check" \
+  | ggrep --line-buffered --invert-match "slow event loop" \
+  | ggrep --line-buffered --invert-match "process.memoryUsage" \
+  | ggrep --line-buffered --only-matching "\{.*" \
+  | bunyan --output short

develop/bin/shell

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose exec -it "$@" /bin/bash

develop/bin/up

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose up --build --detach "$@"

develop/dev.env

@@ -0,0 +1,18 @@
+CHAT_HOST=chat
+CLSI_HOST=clsi
+CONTACTS_HOST=contacts
+DOCSTORE_HOST=docstore
+DOCUMENT_UPDATER_HOST=document-updater
+FILESTORE_HOST=filestore
+GRACEFUL_SHUTDOWN_DELAY=0
+LISTEN_ADDRESS=0.0.0.0
+MONGO_HOST=mongo
+NOTIFICATIONS_HOST=notifications
+REALTIME_HOST=real-time
+REDIS_HOST=redis
+SPELLING_HOST=spelling
+TRACK_CHANGES_HOST=track-changes
+WEBPACK_HOST=webpack
+WEB_API_PASSWORD=sharelatex
+WEB_API_USER=sharelatex
+WEB_HOST=web

develop/docker-compose.dev.yml

@@ -0,0 +1,125 @@
+services:
+  clsi:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9230:9229"
+    volumes:
+      - ../services/clsi/app:/overleaf/services/clsi/app
+      - ../services/clsi/app.js:/overleaf/services/clsi/app.js
+      - ../services/clsi/config:/overleaf/services/clsi/config
+
+  chat:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9231:9229"
+    volumes:
+      - ../services/chat/app:/overleaf/services/chat/app
+      - ../services/chat/app.js:/overleaf/services/chat/app.js
+      - ../services/chat/config:/overleaf/services/chat/config
+
+  contacts:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9232:9229"
+    volumes:
+      - ../services/contacts/app:/overleaf/services/contacts/app
+      - ../services/contacts/app.js:/overleaf/services/contacts/app.js
+      - ../services/contacts/config:/overleaf/services/contacts/config
+
+  docstore:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9233:9229"
+    volumes:
+      - ../services/docstore/app:/overleaf/services/docstore/app
+      - ../services/docstore/app.js:/overleaf/services/docstore/app.js
+      - ../services/docstore/config:/overleaf/services/docstore/config
+
+  document-updater:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9234:9229"
+    volumes:
+      - ../services/document-updater/app:/overleaf/services/document-updater/app
+      - ../services/document-updater/app.js:/overleaf/services/document-updater/app.js
+      - ../services/document-updater/config:/overleaf/services/document-updater/config
+
+  filestore:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9235:9229"
+    volumes:
+      - ../services/filestore/app:/overleaf/services/filestore/app
+      - ../services/filestore/app.js:/overleaf/services/filestore/app.js
+      - ../services/filestore/config:/overleaf/services/filestore/config
+
+  notifications:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9236:9229"
+    volumes:
+      - ../services/notifications/app:/overleaf/services/notifications/app
+      - ../services/notifications/app.js:/overleaf/services/notifications/app.js
+      - ../services/notifications/config:/overleaf/services/notifications/config
+
+  real-time:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9237:9229"
+    volumes:
+      - ../services/real-time/app:/overleaf/services/real-time/app
+      - ../services/real-time/app.js:/overleaf/services/real-time/app.js
+      - ../services/real-time/config:/overleaf/services/real-time/config
+
+  track-changes:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9238:9229"
+    volumes:
+      - ../services/track-changes/app:/overleaf/services/track-changes/app
+      - ../services/track-changes/app.js:/overleaf/services/track-changes/app.js
+      - ../services/track-changes/config:/overleaf/services/track-changes/config
+
+  web:
+    command: ["npm", "run", "nodemon"]
+    environment:
+      - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9229:9229"
+    volumes:
+      - ../services/web/app:/overleaf/services/web/app
+      - ../services/web/app.js:/overleaf/services/web/app.js
+      - ../services/web/config:/overleaf/services/web/config
+      - ../services/web/locales:/overleaf/services/web/locales
+      - ../services/web/modules:/overleaf/services/web/modules
+      - ../services/web/public:/overleaf/services/web/public
+
+  webpack:
+    volumes:
+      - ../services/web/app:/overleaf/services/web/app
+      - ../services/web/config:/overleaf/services/web/config
+      - ../services/web/frontend:/overleaf/services/web/frontend
+      - ../services/web/locales:/overleaf/services/web/locales
+      - ../services/web/modules:/overleaf/services/web/modules
+      - ../services/web/public:/overleaf/services/web/public
+      - ../services/web/transform:/overleaf/services/web/transform
+      - ../services/web/types:/overleaf/services/web/types
+      - ../services/web/webpack-plugins:/overleaf/services/web/webpack-plugins

develop/docker-compose.yml

@@ -0,0 +1,178 @@
+volumes:
+  clsi-cache:
+  clsi-output:
+  filestore-public-files:
+  filestore-template-files:
+  filestore-uploads:
+  filestore-user-files:
+  mongo-data:
+  npm-cache:
+  redis-data:
+  sharelatex-data:
+  spelling-cache:
+  web-data:
+
+services:
+  chat:
+    build:
+      context: ..
+      dockerfile: services/chat/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  clsi:
+    build:
+      context: ..
+      dockerfile: services/clsi/Dockerfile
+    env_file:
+      - dev.env
+    environment:
+      - DOCKER_RUNNER=true
+      - TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full
+      - COMPILES_HOST_DIR=${PWD}/compiles
+    user: root
+    volumes:
+      - npm-cache:/root/.npm
+      - ${PWD}/compiles:/overleaf/services/clsi/compiles
+      - ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
+      - clsi-cache:/overleaf/services/clsi/cache
+      - clsi-output:/overleaf/services/clsi/output
+
+  contacts:
+    build:
+      context: ..
+      dockerfile: services/contacts/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  docstore:
+    build:
+      context: ..
+      dockerfile: services/docstore/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  document-updater:
+    build:
+      context: ..
+      dockerfile: services/document-updater/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  filestore:
+    build:
+      context: ..
+      dockerfile: services/filestore/Dockerfile
+    env_file:
+      - dev.env
+#    environment:
+#      - ENABLE_CONVERSIONS=true
+    volumes:
+      - npm-cache:/root/.npm
+      - filestore-public-files:/overleaf/services/filestore/public_files
+      - filestore-template-files:/overleaf/services/filestore/template_files
+      - filestore-uploads:/overleaf/services/filestore/uploads
+      - filestore-user-files:/overleaf/services/filestore/user_files
+
+  mongo:
+    image: mongo:4.4
+    ports:
+      - "127.0.0.1:27017:27017" # for debugging
+    volumes:
+      - npm-cache:/root/.npm
+      - mongo-data:/data/db
+
+  notifications:
+    build:
+      context: ..
+      dockerfile: services/notifications/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  real-time:
+    build:
+      context: ..
+      dockerfile: services/real-time/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  redis:
+    image: redis:5
+    ports:
+      - "127.0.0.1:6379:6379" # for debugging
+    volumes:
+      - redis-data:/data
+
+  spelling:
+    build:
+      context: ..
+      dockerfile: services/spelling/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+      - spelling-cache:/overleaf/services/spelling/cache
+
+  track-changes:
+    build:
+      context: ..
+      dockerfile: services/track-changes/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - npm-cache:/root/.npm
+
+  web:
+    build:
+      context: ..
+      dockerfile: services/web/Dockerfile
+      target: dev
+    env_file:
+      - dev.env
+    environment:
+      - APP_NAME=Overleaf Community Edition
+      - ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
+      - EMAIL_CONFIRMATION_DISABLED=true
+      - NODE_ENV=development
+      - SHARELATEX_ALLOW_PUBLIC_ACCESS=true
+    command: ["node", "app.js"]
+    volumes:
+      - npm-cache:/root/.npm
+      - sharelatex-data:/var/lib/sharelatex
+      - web-data:/overleaf/services/web/data
+    depends_on:
+      - mongo
+      - redis
+      - chat
+      - clsi
+      - contacts
+      - docstore
+      - document-updater
+      - filestore
+      - notifications
+      - real-time
+      - spelling
+
+  webpack:
+    build:
+      context: ..
+      dockerfile: services/web/Dockerfile
+      target: dev
+    command: ["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"]
+    ports:
+      - "127.0.0.1:80:3808"
+    volumes:
+      - npm-cache:/root/.npm
+      - ./webpack.config.dev-env.js:/overleaf/services/web/webpack.config.dev-env.js

develop/texlive/Dockerfile

@@ -0,0 +1,8 @@
+FROM debian:testing-slim
+
+RUN apt-get update
+RUN apt-cache depends texlive-full | grep "Depends: " | grep -v -- "-doc" | grep -v -- "-lang-" | sed 's/Depends: //' | xargs apt-get install -y --no-install-recommends
+RUN apt-get install -y --no-install-recommends fontconfig inkscape pandoc python3-pygments
+
+RUN useradd tex
+USER tex

develop/webpack.config.dev-env.js

@@ -0,0 +1,23 @@
+const { merge } = require('webpack-merge')
+
+const base = require('./webpack.config.dev')
+
+module.exports = merge(base, {
+  devServer: {
+    allowedHosts: 'auto',
+    devMiddleware: {
+      index: false,
+    },
+    proxy: [
+      {
+        context: '/socket.io/**',
+        target: 'http://real-time:3026',
+        ws: true,
+      },
+      {
+        context: ['!**/*.js', '!**/*.css', '!**/*.json'],
+        target: 'http://web:3000',
+      },
+    ],
+  },
+})

patches/README.md

@@ -0,0 +1 @@
+The patches in this folder are applied by `patch-package` to dependencies, particularly those which need changes that are difficult to apply upstream.

patches/ngcomponent+4.1.0.patch

@@ -0,0 +1,9 @@
+diff --git a/node_modules/ngcomponent/index.ts b/node_modules/ngcomponent/index.ts
+index 5fe33c5..8e1c6fc 100644
+--- a/node_modules/ngcomponent/index.ts
++++ b/node_modules/ngcomponent/index.ts
+@@ -1,3 +1,4 @@
++// @ts-nocheck
+ import { IChangesObject } from 'angular'
+ import assign = require('lodash/assign')
+ import mapValues = require('lodash/mapValues')

patches/react2angular+4.0.6.patch

@@ -0,0 +1,9 @@
+diff --git a/node_modules/react2angular/index.tsx b/node_modules/react2angular/index.tsx
+index 5cee831..a07e040 100644
+--- a/node_modules/react2angular/index.tsx
++++ b/node_modules/react2angular/index.tsx
+@@ -1,3 +1,4 @@
++// @ts-nocheck
+ import { IAugmentedJQuery, IComponentOptions } from 'angular'
+ import fromPairs = require('lodash.frompairs')
+ import NgComponent from 'ngcomponent'