|
|
@@ -1,164 +1,196 @@
|
|
|
-Metrics = require "metrics-sharelatex"
|
|
|
-Metrics.initialize("doc-updater")
|
|
|
-
|
|
|
-express = require('express')
|
|
|
-http = require("http")
|
|
|
-Settings = require('settings-sharelatex')
|
|
|
-logger = require('logger-sharelatex')
|
|
|
-logger.initialize("document-updater")
|
|
|
-
|
|
|
-logger.logger.addSerializers(require("./app/js/LoggerSerializers"))
|
|
|
-
|
|
|
-if Settings.sentry?.dsn?
|
|
|
- logger.initializeErrorReporting(Settings.sentry.dsn)
|
|
|
-
|
|
|
-RedisManager = require('./app/js/RedisManager')
|
|
|
-DispatchManager = require('./app/js/DispatchManager')
|
|
|
-DeleteQueueManager = require('./app/js/DeleteQueueManager')
|
|
|
-Errors = require "./app/js/Errors"
|
|
|
-HttpController = require "./app/js/HttpController"
|
|
|
-mongojs = require "./app/js/mongojs"
|
|
|
-async = require "async"
|
|
|
-
|
|
|
-Path = require "path"
|
|
|
-bodyParser = require "body-parser"
|
|
|
-
|
|
|
-Metrics.mongodb.monitor(Path.resolve(__dirname + "/node_modules/mongojs/node_modules/mongodb"), logger)
|
|
|
-Metrics.event_loop.monitor(logger, 100)
|
|
|
-
|
|
|
-app = express()
|
|
|
+/*
|
|
|
+ * decaffeinate suggestions:
|
|
|
+ * DS101: Remove unnecessary use of Array.from
|
|
|
+ * DS102: Remove unnecessary code created because of implicit returns
|
|
|
+ * DS103: Rewrite code to no longer use __guard__
|
|
|
+ * DS207: Consider shorter variations of null checks
|
|
|
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
|
|
+ */
|
|
|
+const Metrics = require("metrics-sharelatex");
|
|
|
+Metrics.initialize("doc-updater");
|
|
|
+
|
|
|
+const express = require('express');
|
|
|
+const http = require("http");
|
|
|
+const Settings = require('settings-sharelatex');
|
|
|
+const logger = require('logger-sharelatex');
|
|
|
+logger.initialize("document-updater");
|
|
|
+
|
|
|
+logger.logger.addSerializers(require("./app/js/LoggerSerializers"));
|
|
|
+
|
|
|
+if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
|
|
|
+ logger.initializeErrorReporting(Settings.sentry.dsn);
|
|
|
+}
|
|
|
+
|
|
|
+const RedisManager = require('./app/js/RedisManager');
|
|
|
+const DispatchManager = require('./app/js/DispatchManager');
|
|
|
+const DeleteQueueManager = require('./app/js/DeleteQueueManager');
|
|
|
+const Errors = require("./app/js/Errors");
|
|
|
+const HttpController = require("./app/js/HttpController");
|
|
|
+const mongojs = require("./app/js/mongojs");
|
|
|
+const async = require("async");
|
|
|
+
|
|
|
+const Path = require("path");
|
|
|
+const bodyParser = require("body-parser");
|
|
|
+
|
|
|
+Metrics.mongodb.monitor(Path.resolve(__dirname + "/node_modules/mongojs/node_modules/mongodb"), logger);
|
|
|
+Metrics.event_loop.monitor(logger, 100);
|
|
|
+
|
|
|
+const app = express();
|
|
|
app.use(Metrics.http.monitor(logger));
|
|
|
-app.use bodyParser.json({limit: (Settings.max_doc_length + 64 * 1024)})
|
|
|
-Metrics.injectMetricsRoute(app)
|
|
|
-
|
|
|
-DispatchManager.createAndStartDispatchers(Settings.dispatcherCount || 10)
|
|
|
-
|
|
|
-app.param 'project_id', (req, res, next, project_id) ->
|
|
|
- if project_id?.match /^[0-9a-f]{24}$/
|
|
|
- next()
|
|
|
- else
|
|
|
- next new Error("invalid project id")
|
|
|
-
|
|
|
-app.param 'doc_id', (req, res, next, doc_id) ->
|
|
|
- if doc_id?.match /^[0-9a-f]{24}$/
|
|
|
- next()
|
|
|
- else
|
|
|
- next new Error("invalid doc id")
|
|
|
-
|
|
|
-app.get '/project/:project_id/doc/:doc_id', HttpController.getDoc
|
|
|
-# temporarily keep the GET method for backwards compatibility
|
|
|
-app.get '/project/:project_id/doc', HttpController.getProjectDocsAndFlushIfOld
|
|
|
-# will migrate to the POST method of get_and_flush_if_old instead
|
|
|
-app.post '/project/:project_id/get_and_flush_if_old', HttpController.getProjectDocsAndFlushIfOld
|
|
|
-app.post '/project/:project_id/clearState', HttpController.clearProjectState
|
|
|
-app.post '/project/:project_id/doc/:doc_id', HttpController.setDoc
|
|
|
-app.post '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLoaded
|
|
|
-app.delete '/project/:project_id/doc/:doc_id', HttpController.deleteDoc
|
|
|
-app.delete '/project/:project_id', HttpController.deleteProject
|
|
|
-app.delete '/project', HttpController.deleteMultipleProjects
|
|
|
-app.post '/project/:project_id', HttpController.updateProject
|
|
|
-app.post '/project/:project_id/history/resync', HttpController.resyncProjectHistory
|
|
|
-app.post '/project/:project_id/flush', HttpController.flushProject
|
|
|
-app.post '/project/:project_id/doc/:doc_id/change/:change_id/accept', HttpController.acceptChanges
|
|
|
-app.post '/project/:project_id/doc/:doc_id/change/accept', HttpController.acceptChanges
|
|
|
-app.delete '/project/:project_id/doc/:doc_id/comment/:comment_id', HttpController.deleteComment
|
|
|
-
|
|
|
-app.get '/flush_all_projects', HttpController.flushAllProjects
|
|
|
-app.get '/flush_queued_projects', HttpController.flushQueuedProjects
|
|
|
-
|
|
|
-app.get '/total', (req, res)->
|
|
|
- timer = new Metrics.Timer("http.allDocList")
|
|
|
- RedisManager.getCountOfDocsInMemory (err, count)->
|
|
|
- timer.done()
|
|
|
- res.send {total:count}
|
|
|
-
|
|
|
-app.get '/status', (req, res)->
|
|
|
- if Settings.shuttingDown
|
|
|
- res.sendStatus 503 # Service unavailable
|
|
|
- else
|
|
|
- res.send('document updater is alive')
|
|
|
-
|
|
|
-pubsubClient = require("redis-sharelatex").createClient(Settings.redis.pubsub)
|
|
|
-app.get "/health_check/redis", (req, res, next) ->
|
|
|
- pubsubClient.healthCheck (error) ->
|
|
|
- if error?
|
|
|
- logger.err {err: error}, "failed redis health check"
|
|
|
- res.sendStatus 500
|
|
|
- else
|
|
|
- res.sendStatus 200
|
|
|
-
|
|
|
-docUpdaterRedisClient = require("redis-sharelatex").createClient(Settings.redis.documentupdater)
|
|
|
-app.get "/health_check/redis_cluster", (req, res, next) ->
|
|
|
- docUpdaterRedisClient.healthCheck (error) ->
|
|
|
- if error?
|
|
|
- logger.err {err: error}, "failed redis cluster health check"
|
|
|
- res.sendStatus 500
|
|
|
- else
|
|
|
- res.sendStatus 200
|
|
|
-
|
|
|
-app.get "/health_check", (req, res, next) ->
|
|
|
- async.series [
|
|
|
- (cb) ->
|
|
|
- pubsubClient.healthCheck (error) ->
|
|
|
- if error?
|
|
|
- logger.err {err: error}, "failed redis health check"
|
|
|
- cb(error)
|
|
|
- (cb) ->
|
|
|
- docUpdaterRedisClient.healthCheck (error) ->
|
|
|
- if error?
|
|
|
- logger.err {err: error}, "failed redis cluster health check"
|
|
|
- cb(error)
|
|
|
- (cb) ->
|
|
|
- mongojs.healthCheck (error) ->
|
|
|
- if error?
|
|
|
- logger.err {err: error}, "failed mongo health check"
|
|
|
- cb(error)
|
|
|
- ] , (error) ->
|
|
|
- if error?
|
|
|
- res.sendStatus 500
|
|
|
- else
|
|
|
- res.sendStatus 200
|
|
|
-
|
|
|
-app.use (error, req, res, next) ->
|
|
|
- if error instanceof Errors.NotFoundError
|
|
|
- res.sendStatus 404
|
|
|
- else if error instanceof Errors.OpRangeNotAvailableError
|
|
|
- res.sendStatus 422 # Unprocessable Entity
|
|
|
- else if error.statusCode is 413
|
|
|
- res.status(413).send("request entity too large")
|
|
|
- else
|
|
|
- logger.error err: error, req: req, "request errored"
|
|
|
- res.status(500).send("Oops, something went wrong")
|
|
|
-
|
|
|
-shutdownCleanly = (signal) ->
|
|
|
- return () ->
|
|
|
- logger.log signal: signal, "received interrupt, cleaning up"
|
|
|
- Settings.shuttingDown = true
|
|
|
- setTimeout () ->
|
|
|
- logger.log signal: signal, "shutting down"
|
|
|
- process.exit()
|
|
|
- , 10000
|
|
|
-
|
|
|
-watchForEvent = (eventName)->
|
|
|
- docUpdaterRedisClient.on eventName, (e)->
|
|
|
- console.log "redis event: #{eventName} #{e}"
|
|
|
-
|
|
|
-events = ["connect", "ready", "error", "close", "reconnecting", "end"]
|
|
|
-for eventName in events
|
|
|
- watchForEvent(eventName)
|
|
|
-
|
|
|
-
|
|
|
-port = Settings.internal?.documentupdater?.port or Settings.apis?.documentupdater?.port or 3003
|
|
|
-host = Settings.internal.documentupdater.host or "localhost"
|
|
|
-if !module.parent # Called directly
|
|
|
- app.listen port, host, ->
|
|
|
- logger.info "Document-updater starting up, listening on #{host}:#{port}"
|
|
|
- if Settings.continuousBackgroundFlush
|
|
|
- logger.info "Starting continuous background flush"
|
|
|
- DeleteQueueManager.startBackgroundFlush()
|
|
|
-
|
|
|
-module.exports = app
|
|
|
-
|
|
|
-for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']
|
|
|
- process.on signal, shutdownCleanly(signal)
|
|
|
-
|
|
|
+app.use(bodyParser.json({limit: (Settings.max_doc_length + (64 * 1024))}));
|
|
|
+Metrics.injectMetricsRoute(app);
|
|
|
+
|
|
|
+DispatchManager.createAndStartDispatchers(Settings.dispatcherCount || 10);
|
|
|
+
|
|
|
+app.param('project_id', function(req, res, next, project_id) {
|
|
|
+ if ((project_id != null ? project_id.match(/^[0-9a-f]{24}$/) : undefined)) {
|
|
|
+ return next();
|
|
|
+ } else {
|
|
|
+ return next(new Error("invalid project id"));
|
|
|
+ }
|
|
|
+});
|
|
|
+
|
|
|
+app.param('doc_id', function(req, res, next, doc_id) {
|
|
|
+ if ((doc_id != null ? doc_id.match(/^[0-9a-f]{24}$/) : undefined)) {
|
|
|
+ return next();
|
|
|
+ } else {
|
|
|
+ return next(new Error("invalid doc id"));
|
|
|
+ }
|
|
|
+});
|
|
|
+
|
|
|
+app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc);
|
|
|
+// temporarily keep the GET method for backwards compatibility
|
|
|
+app.get('/project/:project_id/doc', HttpController.getProjectDocsAndFlushIfOld);
|
|
|
+// will migrate to the POST method of get_and_flush_if_old instead
|
|
|
+app.post('/project/:project_id/get_and_flush_if_old', HttpController.getProjectDocsAndFlushIfOld);
|
|
|
+app.post('/project/:project_id/clearState', HttpController.clearProjectState);
|
|
|
+app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc);
|
|
|
+app.post('/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLoaded);
|
|
|
+app.delete('/project/:project_id/doc/:doc_id', HttpController.deleteDoc);
|
|
|
+app.delete('/project/:project_id', HttpController.deleteProject);
|
|
|
+app.delete('/project', HttpController.deleteMultipleProjects);
|
|
|
+app.post('/project/:project_id', HttpController.updateProject);
|
|
|
+app.post('/project/:project_id/history/resync', HttpController.resyncProjectHistory);
|
|
|
+app.post('/project/:project_id/flush', HttpController.flushProject);
|
|
|
+app.post('/project/:project_id/doc/:doc_id/change/:change_id/accept', HttpController.acceptChanges);
|
|
|
+app.post('/project/:project_id/doc/:doc_id/change/accept', HttpController.acceptChanges);
|
|
|
+app.delete('/project/:project_id/doc/:doc_id/comment/:comment_id', HttpController.deleteComment);
|
|
|
+
|
|
|
+app.get('/flush_all_projects', HttpController.flushAllProjects);
|
|
|
+app.get('/flush_queued_projects', HttpController.flushQueuedProjects);
|
|
|
+
|
|
|
+app.get('/total', function(req, res){
|
|
|
+ const timer = new Metrics.Timer("http.allDocList");
|
|
|
+ return RedisManager.getCountOfDocsInMemory(function(err, count){
|
|
|
+ timer.done();
|
|
|
+ return res.send({total:count});});
|
|
|
+});
|
|
|
+
|
|
|
+app.get('/status', function(req, res){
|
|
|
+ if (Settings.shuttingDown) {
|
|
|
+ return res.sendStatus(503); // Service unavailable
|
|
|
+ } else {
|
|
|
+ return res.send('document updater is alive');
|
|
|
+ }
|
|
|
+});
|
|
|
+
|
|
|
+const pubsubClient = require("redis-sharelatex").createClient(Settings.redis.pubsub);
|
|
|
+app.get("/health_check/redis", (req, res, next) => pubsubClient.healthCheck(function(error) {
|
|
|
+ if (error != null) {
|
|
|
+ logger.err({err: error}, "failed redis health check");
|
|
|
+ return res.sendStatus(500);
|
|
|
+ } else {
|
|
|
+ return res.sendStatus(200);
|
|
|
+ }
|
|
|
+}));
|
|
|
+
|
|
|
+const docUpdaterRedisClient = require("redis-sharelatex").createClient(Settings.redis.documentupdater);
|
|
|
+app.get("/health_check/redis_cluster", (req, res, next) => docUpdaterRedisClient.healthCheck(function(error) {
|
|
|
+ if (error != null) {
|
|
|
+ logger.err({err: error}, "failed redis cluster health check");
|
|
|
+ return res.sendStatus(500);
|
|
|
+ } else {
|
|
|
+ return res.sendStatus(200);
|
|
|
+ }
|
|
|
+}));
|
|
|
+
|
|
|
+app.get("/health_check", (req, res, next) => async.series([
|
|
|
+ cb => pubsubClient.healthCheck(function(error) {
|
|
|
+ if (error != null) {
|
|
|
+ logger.err({err: error}, "failed redis health check");
|
|
|
+ }
|
|
|
+ return cb(error);
|
|
|
+ }),
|
|
|
+ cb => docUpdaterRedisClient.healthCheck(function(error) {
|
|
|
+ if (error != null) {
|
|
|
+ logger.err({err: error}, "failed redis cluster health check");
|
|
|
+ }
|
|
|
+ return cb(error);
|
|
|
+ }),
|
|
|
+ cb => mongojs.healthCheck(function(error) {
|
|
|
+ if (error != null) {
|
|
|
+ logger.err({err: error}, "failed mongo health check");
|
|
|
+ }
|
|
|
+ return cb(error);
|
|
|
+ })
|
|
|
+] , function(error) {
|
|
|
+ if (error != null) {
|
|
|
+ return res.sendStatus(500);
|
|
|
+ } else {
|
|
|
+ return res.sendStatus(200);
|
|
|
+ }
|
|
|
+}));
|
|
|
+
|
|
|
+app.use(function(error, req, res, next) {
|
|
|
+ if (error instanceof Errors.NotFoundError) {
|
|
|
+ return res.sendStatus(404);
|
|
|
+ } else if (error instanceof Errors.OpRangeNotAvailableError) {
|
|
|
+ return res.sendStatus(422); // Unprocessable Entity
|
|
|
+ } else if (error.statusCode === 413) {
|
|
|
+ return res.status(413).send("request entity too large");
|
|
|
+ } else {
|
|
|
+ logger.error({err: error, req}, "request errored");
|
|
|
+ return res.status(500).send("Oops, something went wrong");
|
|
|
+ }
|
|
|
+});
|
|
|
+
|
|
|
+const shutdownCleanly = signal => (function() {
|
|
|
+ logger.log({signal}, "received interrupt, cleaning up");
|
|
|
+ Settings.shuttingDown = true;
|
|
|
+ return setTimeout(function() {
|
|
|
+ logger.log({signal}, "shutting down");
|
|
|
+ return process.exit();
|
|
|
+ }
|
|
|
+ , 10000);
|
|
|
+});
|
|
|
+
|
|
|
+const watchForEvent = eventName => docUpdaterRedisClient.on(eventName, e => console.log(`redis event: ${eventName} ${e}`));
|
|
|
+
|
|
|
+const events = ["connect", "ready", "error", "close", "reconnecting", "end"];
|
|
|
+for (let eventName of Array.from(events)) {
|
|
|
+ watchForEvent(eventName);
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+const port = __guard__(Settings.internal != null ? Settings.internal.documentupdater : undefined, x => x.port) || __guard__(Settings.apis != null ? Settings.apis.documentupdater : undefined, x1 => x1.port) || 3003;
|
|
|
+const host = Settings.internal.documentupdater.host || "localhost";
|
|
|
+if (!module.parent) { // Called directly
|
|
|
+ app.listen(port, host, function() {
|
|
|
+ logger.info(`Document-updater starting up, listening on ${host}:${port}`);
|
|
|
+ if (Settings.continuousBackgroundFlush) {
|
|
|
+ logger.info("Starting continuous background flush");
|
|
|
+ return DeleteQueueManager.startBackgroundFlush();
|
|
|
+ }
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+module.exports = app;
|
|
|
+
|
|
|
+for (let signal of ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']) {
|
|
|
+ process.on(signal, shutdownCleanly(signal));
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+function __guard__(value, transform) {
|
|
|
+ return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
|
|
+}
|