Merge pull request #5351 from overleaf/jpa-clsi-drop-sqlite

[clsi] goodbye sqlite db

GitOrigin-RevId: e7f16920c44e74a425b92884b48a60272dfb015b

services/clsi/.dockerignore

@@ -7,5 +7,4 @@ gitrev
 nodemon.json
 cache/
 compiles/
-db/
 output/

services/clsi/.gitignore

@@ -7,9 +7,6 @@ output
 *~
 cache
 .vagrant
-db.sqlite
-db.sqlite-wal
-db.sqlite-shm
 config/*
 npm-debug.log
 

services/clsi/Dockerfile

@@ -22,7 +22,7 @@ COPY . /app
 FROM base
 
 COPY --from=app /app /app
-RUN mkdir -p cache compiles db output \
-&&  chown node:node cache compiles db output
+RUN mkdir -p cache compiles output \
+&&  chown node:node cache compiles output
 
 CMD ["node", "--expose-gc", "app.js"]

services/clsi/README.md

@@ -30,7 +30,6 @@ The CLSI can be configured through the following environment variables:
 * `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds
 * `SENTRY_DSN` - Sentry [Data Source Name](https://docs.sentry.io/product/sentry-basics/dsn-explainer/)
 * `SMOKE_TEST` - Whether to run smoke tests
-* `SQLITE_PATH` - Path to SQLite database
 * `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1`
 * `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `gcr.io/overleaf-ops`
 * `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex`

services/clsi/app.js

@@ -30,7 +30,6 @@ const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager')
 const OutputCacheManager = require('./app/js/OutputCacheManager')
 const ContentCacheManager = require('./app/js/ContentCacheManager')
 
-require('./app/js/db').sync()
 ProjectPersistenceManager.init()
 
 const express = require('express')

services/clsi/app/js/db.js

@@ -1,67 +0,0 @@
-/* eslint-disable
-    no-console,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-const Sequelize = require('sequelize')
-const Settings = require('@overleaf/settings')
-const _ = require('lodash')
-const logger = require('logger-sharelatex')
-
-const options = _.extend({ logging: false }, Settings.mysql.clsi)
-
-logger.log({ dbPath: Settings.mysql.clsi.storage }, 'connecting to db')
-
-const sequelize = new Sequelize(
-  Settings.mysql.clsi.database,
-  Settings.mysql.clsi.username,
-  Settings.mysql.clsi.password,
-  options
-)
-
-if (Settings.mysql.clsi.dialect === 'sqlite') {
-  logger.log('running PRAGMA journal_mode=WAL;')
-  sequelize.query('PRAGMA journal_mode=WAL;')
-  sequelize.query('PRAGMA synchronous=OFF;')
-  sequelize.query('PRAGMA read_uncommitted = true;')
-}
-
-module.exports = {
-  UrlCache: sequelize.define(
-    'UrlCache',
-    {
-      url: Sequelize.STRING,
-      project_id: Sequelize.STRING,
-      lastModified: Sequelize.DATE,
-    },
-    {
-      indexes: [{ fields: ['url', 'project_id'] }, { fields: ['project_id'] }],
-    }
-  ),
-
-  Project: sequelize.define(
-    'Project',
-    {
-      project_id: { type: Sequelize.STRING, primaryKey: true },
-      lastAccessed: Sequelize.DATE,
-    },
-    {
-      indexes: [{ fields: ['lastAccessed'] }],
-    }
-  ),
-
-  op: Sequelize.Op,
-
-  sync() {
-    logger.log({ dbPath: Settings.mysql.clsi.storage }, 'syncing db schema')
-    return sequelize
-      .sync()
-      .then(() => logger.log('db sync complete'))
-      .catch(err => console.log(err, 'error syncing'))
-  },
-}

services/clsi/buildscript.txt

@@ -1,5 +1,5 @@
 clsi
---data-dirs=cache,compiles,db,output
+--data-dirs=cache,compiles,output
 --dependencies=
 --docker-repos=gcr.io/overleaf-ops
 --env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true"

services/clsi/config/settings.defaults.js

@@ -1,25 +1,6 @@
 const Path = require('path')
 
 module.exports = {
-  // Options are passed to Sequelize.
-  // See http://sequelizejs.com/documentation#usage-options for details
-  mysql: {
-    clsi: {
-      database: 'clsi',
-      username: 'clsi',
-      dialect: 'sqlite',
-      storage:
-        process.env.SQLITE_PATH || Path.resolve(__dirname, '../db/db.sqlite'),
-      pool: {
-        max: 1,
-        min: 1,
-      },
-      retry: {
-        max: 10,
-      },
-    },
-  },
-
   compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
 
   processLifespanLimitMs:

services/clsi/db/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

services/clsi/docker-compose-config.yml

@@ -23,7 +23,6 @@ services:
       SHARELATEX_CONFIG: /app/config/settings.defaults.js
       DOCKER_RUNNER: "true"
       COMPILES_HOST_DIR: $PWD/compiles
-      SQLITE_PATH: /app/compiles/db.sqlite
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:rw
       - ./compiles:/app/compiles

services/clsi/package-lock.json

@@ -1225,11 +1225,6 @@
         "color-convert": "^1.9.0"
       }
     },
-    "any-promise": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
-      "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
-    },
     "anymatch": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
@@ -1437,11 +1432,6 @@
         }
       }
     },
-    "bluebird": {
-      "version": "3.7.2",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
-    },
     "body-parser": {
       "version": "1.19.0",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
@@ -1848,15 +1838,6 @@
         "mimic-response": "^1.0.0"
       }
     },
-    "cls-bluebird": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.1.0.tgz",
-      "integrity": "sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4=",
-      "requires": {
-        "is-bluebird": "^1.0.2",
-        "shimmer": "^1.1.0"
-      }
-    },
     "code-point-at": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
@@ -2214,11 +2195,6 @@
         "is-obj": "^2.0.0"
       }
     },
-    "dottie": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",
-      "integrity": "sha512-fmrwR04lsniq/uSr8yikThDTrM7epXHBAAjH9TbeH3rEA8tdCO7mRzB9hdmdGyJCxF8KERo9CITcm3kGuoyMhg=="
-    },
     "dtrace-provider": {
       "version": "0.8.8",
       "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz",
@@ -3737,11 +3713,6 @@
       "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
       "dev": true
     },
-    "inflection": {
-      "version": "1.12.0",
-      "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz",
-      "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY="
-    },
     "inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -3792,11 +3763,6 @@
         "binary-extensions": "^2.0.0"
       }
     },
-    "is-bluebird": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz",
-      "integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI="
-    },
     "is-boolean-object": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz",
@@ -4675,15 +4641,8 @@
     "moment": {
       "version": "2.23.0",
       "resolved": "https://registry.npmjs.org/moment/-/moment-2.23.0.tgz",
-      "integrity": "sha1-dZ6kkayX1UusWtd2mW4qWMwbwiU="
-    },
-    "moment-timezone": {
-      "version": "0.5.28",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz",
-      "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==",
-      "requires": {
-        "moment": ">= 2.9.0"
-      }
+      "integrity": "sha1-dZ6kkayX1UusWtd2mW4qWMwbwiU=",
+      "optional": true
     },
     "ms": {
       "version": "2.0.0",
@@ -4725,38 +4684,6 @@
         }
       }
     },
-    "mysql": {
-      "version": "2.18.1",
-      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
-      "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
-      "requires": {
-        "bignumber.js": "9.0.0",
-        "readable-stream": "2.3.7",
-        "safe-buffer": "5.1.2",
-        "sqlstring": "2.3.1"
-      },
-      "dependencies": {
-        "bignumber.js": {
-          "version": "9.0.0",
-          "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
-          "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="
-        },
-        "readable-stream": {
-          "version": "2.3.7",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        }
-      }
-    },
     "nan": {
       "version": "2.14.0",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
@@ -4780,16 +4707,6 @@
       "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
       "optional": true
     },
-    "needle": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz",
-      "integrity": "sha1-UZMb/4JTOxkot9HWngHxsA/9Kk4=",
-      "requires": {
-        "debug": "^2.1.2",
-        "iconv-lite": "^0.4.4",
-        "sax": "^1.2.4"
-      }
-    },
     "negotiator": {
       "version": "0.6.2",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@@ -4835,23 +4752,6 @@
       "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz",
       "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ=="
     },
-    "node-pre-gyp": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
-      "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
-      "requires": {
-        "detect-libc": "^1.0.2",
-        "mkdirp": "^0.5.1",
-        "needle": "^2.2.1",
-        "nopt": "^4.0.1",
-        "npm-packlist": "^1.1.6",
-        "npmlog": "^4.0.2",
-        "rc": "^1.2.7",
-        "rimraf": "^2.6.1",
-        "semver": "^5.3.0",
-        "tar": "^4"
-      }
-    },
     "nodemon": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz",
@@ -5751,14 +5651,6 @@
         "lowercase-keys": "^1.0.0"
       }
     },
-    "retry-as-promised": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz",
-      "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==",
-      "requires": {
-        "any-promise": "^1.3.0"
-      }
-    },
     "retry-request": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz",
@@ -5871,58 +5763,6 @@
         }
       }
     },
-    "sequelize": {
-      "version": "5.21.5",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.5.tgz",
-      "integrity": "sha512-n9hR5K4uQGmBGK/Y/iqewCeSFmKVsd0TRnh0tfoLoAkmXbKC4tpeK96RhKs7d+TTMtrJlgt2TNLVBaAxEwC4iw==",
-      "requires": {
-        "bluebird": "^3.5.0",
-        "cls-bluebird": "^2.1.0",
-        "debug": "^4.1.1",
-        "dottie": "^2.0.0",
-        "inflection": "1.12.0",
-        "lodash": "^4.17.15",
-        "moment": "^2.24.0",
-        "moment-timezone": "^0.5.21",
-        "retry-as-promised": "^3.2.0",
-        "semver": "^6.3.0",
-        "sequelize-pool": "^2.3.0",
-        "toposort-class": "^1.0.1",
-        "uuid": "^3.3.3",
-        "validator": "^10.11.0",
-        "wkx": "^0.4.8"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "moment": {
-          "version": "2.24.0",
-          "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
-          "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
-        },
-        "semver": {
-          "version": "6.3.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
-        }
-      }
-    },
-    "sequelize-pool": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz",
-      "integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA=="
-    },
     "serialize-javascript": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz",
@@ -6139,21 +5979,6 @@
       "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
       "dev": true
     },
-    "sqlite3": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.1.1.tgz",
-      "integrity": "sha512-CvT5XY+MWnn0HkbwVKJAyWEMfzpAPwnTiB3TobA5Mri44SrTovmmh499NPQP+gatkeOipqPlBLel7rn4E/PCQg==",
-      "requires": {
-        "nan": "^2.12.1",
-        "node-pre-gyp": "^0.11.0",
-        "request": "^2.87.0"
-      }
-    },
-    "sqlstring": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
-      "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
-    },
     "ssh2": {
       "version": "0.8.8",
       "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.8.tgz",
@@ -6507,11 +6332,6 @@
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
       "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
     },
-    "toposort-class": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
-      "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
-    },
     "touch": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz",
@@ -6775,11 +6595,6 @@
         "spdx-expression-parse": "^3.0.0"
       }
     },
-    "validator": {
-      "version": "10.11.0",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",
-      "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw=="
-    },
     "vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
@@ -6873,14 +6688,6 @@
         }
       }
     },
-    "wkx": {
-      "version": "0.4.8",
-      "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz",
-      "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==",
-      "requires": {
-        "@types/node": "*"
-      }
-    },
     "word-wrap": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",

services/clsi/package.json

@@ -33,14 +33,11 @@
     "lockfile": "^1.0.4",
     "lodash": "^4.17.21",
     "logger-sharelatex": "^2.2.0",
-    "mysql": "^2.18.1",
-    "workerpool": "^6.1.5",
     "p-limit": "^3.1.0",
     "pdfjs-dist": "^2.7.570",
     "request": "^2.88.2",
     "send": "^0.17.1",
-    "sequelize": "^5.21.5",
-    "sqlite3": "^4.1.1"
+    "workerpool": "^6.1.5"
   },
   "devDependencies": {
     "chai": "^4.2.0",

services/clsi/test/acceptance/scripts/settings.test.js

@@ -1,18 +1,6 @@
 const Path = require('path')
 
 module.exports = {
-  // Options are passed to Sequelize.
-  // See http://sequelizejs.com/documentation#usage-options for details
-  mysql: {
-    clsi: {
-      database: 'clsi',
-      username: 'clsi',
-      password: null,
-      dialect: 'sqlite',
-      storage: Path.resolve('db.sqlite'),
-    },
-  },
-
   path: {
     // eslint-disable-next-line no-path-concat
     compilesDir: Path.resolve(__dirname + '/../../../compiles'),

services/clsi/test/unit/js/ProjectPersistenceManagerTests.js

@@ -35,7 +35,6 @@ describe('ProjectPersistenceManager', function () {
             clsiCacheDir: '/cache',
           },
         }),
-        './db': (this.db = {}),
       },
     })
     this.callback = sinon.stub()