Alf Eaton před 5 roky
rodič
revize
de50e59557

+ 61 - 43
services/document-updater/expire_docops.js

@@ -11,50 +11,68 @@
  * DS207: Consider shorter variations of null checks
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
  */
-const Settings = require("settings-sharelatex");
-const rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater);
-let keys = Settings.redis.documentupdater.key_schema;
-const async = require("async");
-const RedisManager = require("./app/js/RedisManager");
+const Settings = require('settings-sharelatex')
+const rclient = require('@overleaf/redis-wrapper').createClient(
+  Settings.redis.documentupdater
+)
+let keys = Settings.redis.documentupdater.key_schema
+const async = require('async')
+const RedisManager = require('./app/js/RedisManager')
 
 
-const getKeysFromNode = function(node, pattern, callback) {
-	let cursor = 0;  // redis iterator
-	const keySet = {}; // use hash to avoid duplicate results
-	// scan over all keys looking for pattern
-	var doIteration = cb => node.scan(cursor, "MATCH", pattern, "COUNT", 1000, function(error, reply) {
-        if (error != null) { return callback(error); }
-        [cursor, keys] = Array.from(reply);
-        console.log("SCAN", keys.length);
-        for (const key of Array.from(keys)) {
-            keySet[key] = true;
-        }
-        if (cursor === '0') {  // note redis returns string result not numeric
-            return callback(null, Object.keys(keySet));
-        } else {
-            return doIteration();
-        }
-    });
-	return doIteration();
-};
+const getKeysFromNode = function (node, pattern, callback) {
+  let cursor = 0 // redis iterator
+  const keySet = {} // use hash to avoid duplicate results
+  // scan over all keys looking for pattern
+  var doIteration = (cb) =>
+    node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) {
+      if (error != null) {
+        return callback(error)
+      }
+      ;[cursor, keys] = Array.from(reply)
+      console.log('SCAN', keys.length)
+      for (const key of Array.from(keys)) {
+        keySet[key] = true
+      }
+      if (cursor === '0') {
+        // note redis returns string result not numeric
+        return callback(null, Object.keys(keySet))
+      } else {
+        return doIteration()
+      }
+    })
+  return doIteration()
+}
 
 
-const getKeys = function(pattern, callback) {
-	const nodes = (typeof rclient.nodes === 'function' ? rclient.nodes('master') : undefined) || [ rclient ];
-	console.log("GOT NODES", nodes.length);
-	const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb);
-	return async.concatSeries(nodes, doKeyLookupForNode, callback);
-};
+const getKeys = function (pattern, callback) {
+  const nodes = (typeof rclient.nodes === 'function'
+    ? rclient.nodes('master')
+    : undefined) || [rclient]
+  console.log('GOT NODES', nodes.length)
+  const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb)
+  return async.concatSeries(nodes, doKeyLookupForNode, callback)
+}
 
 
-const TTL = 60 * 60; // 1 hour
-const expireDocOps = callback => getKeys(keys.docOps({doc_id: "*"}), (error, keys) => async.mapSeries(keys,
-    function(key, cb) {
-        console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`);
-        return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb);
-    },
-    callback));
+const TTL = 60 * 60 // 1 hour
+const expireDocOps = (callback) =>
+  getKeys(keys.docOps({ doc_id: '*' }), (error, keys) =>
+    async.mapSeries(
+      keys,
+      function (key, cb) {
+        console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`)
+        return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb)
+      },
+      callback
+    )
+  )
 
 
-setTimeout(() => //  Give redis a chance to connect
-expireDocOps(function(error) {
-    if (error != null) { throw error; }
-    return process.exit();
-})
-, 1000);
+setTimeout(
+  () =>
+    //  Give redis a chance to connect
+    expireDocOps(function (error) {
+      if (error != null) {
+        throw error
+      }
+      return process.exit()
+    }),
+  1000
+)

+ 61 - 41
services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js

@@ -10,49 +10,69 @@
  * DS207: Consider shorter variations of null checks
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
  */
-let listenInBackground, sendPings;
-const redis = require("@overleaf/redis-wrapper");
-const rclient1 = redis.createClient({cluster: [{
-	port: "7000",
-	host: "localhost"
-}]});
+let listenInBackground, sendPings
+const redis = require('@overleaf/redis-wrapper')
+const rclient1 = redis.createClient({
+  cluster: [
+    {
+      port: '7000',
+      host: 'localhost'
+    }
+  ]
+})
 
 
-const rclient2 = redis.createClient({cluster: [{
-	port: "7000",
-	host: "localhost"
-}]});
+const rclient2 = redis.createClient({
+  cluster: [
+    {
+      port: '7000',
+      host: 'localhost'
+    }
+  ]
+})
 
 
-let counter = 0;
-const sendPing = function(cb) {
-	if (cb == null) { cb = function() {}; }
-	return rclient1.rpush("test-blpop", counter, (error) => {
-		if (error != null) { console.error("[SENDING ERROR]", error.message); }
-		if ((error == null)) {
-			counter += 1;
-		}
-		return cb();
-	});
-};
+let counter = 0
+const sendPing = function (cb) {
+  if (cb == null) {
+    cb = function () {}
+  }
+  return rclient1.rpush('test-blpop', counter, (error) => {
+    if (error != null) {
+      console.error('[SENDING ERROR]', error.message)
+    }
+    if (error == null) {
+      counter += 1
+    }
+    return cb()
+  })
+}
 
 
-let previous = null;
-const listenForPing = cb => rclient2.blpop("test-blpop", 200, (error, result) => {
-    if (error != null) { return cb(error); }
-    let [key, value] = Array.from(result);
-    value = parseInt(value, 10);
-    if ((value % 10) === 0) {
-        console.log(".");
-    }
-    if ((previous != null) && (value !== (previous + 1))) {
-        error = new Error(`Counter not in order. Got ${value}, expected ${previous + 1}`);
-    }
-    previous = value;
-    return cb(error, value);
-});
+let previous = null
+const listenForPing = (cb) =>
+  rclient2.blpop('test-blpop', 200, (error, result) => {
+    if (error != null) {
+      return cb(error)
+    }
+    let [key, value] = Array.from(result)
+    value = parseInt(value, 10)
+    if (value % 10 === 0) {
+      console.log('.')
+    }
+    if (previous != null && value !== previous + 1) {
+      error = new Error(
+        `Counter not in order. Got ${value}, expected ${previous + 1}`
+      )
+    }
+    previous = value
+    return cb(error, value)
+  })
 
 
-const PING_DELAY = 100;
-(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))();
+const PING_DELAY = 100
+;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()
 
 
-(listenInBackground = () => listenForPing((error, value) => {
-    if (error) { console.error("[RECEIVING ERROR]", error.message); }
-    return setTimeout(listenInBackground);
-}))();
+;(listenInBackground = () =>
+  listenForPing((error, value) => {
+    if (error) {
+      console.error('[RECEIVING ERROR]', error.message)
+    }
+    return setTimeout(listenInBackground)
+  }))()

+ 50 - 35
services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js

@@ -9,42 +9,57 @@
  * DS207: Consider shorter variations of null checks
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
  */
-let sendPings;
-const redis = require("@overleaf/redis-wrapper");
-const rclient1 = redis.createClient({cluster: [{
-	port: "7000",
-	host: "localhost"
-}]});
+let sendPings
+const redis = require('@overleaf/redis-wrapper')
+const rclient1 = redis.createClient({
+  cluster: [
+    {
+      port: '7000',
+      host: 'localhost'
+    }
+  ]
+})
 
 
-const rclient2 = redis.createClient({cluster: [{
-	port: "7000",
-	host: "localhost"
-}]});
+const rclient2 = redis.createClient({
+  cluster: [
+    {
+      port: '7000',
+      host: 'localhost'
+    }
+  ]
+})
 
 
-let counter = 0;
-const sendPing = function(cb) {
-	if (cb == null) { cb = function() {}; }
-	return rclient1.publish("test-pubsub", counter, (error) => {
-		if (error != null) { console.error("[SENDING ERROR]", error.message); }
-		if ((error == null)) {
-			counter += 1;
-		}
-		return cb();
-	});
-};
+let counter = 0
+const sendPing = function (cb) {
+  if (cb == null) {
+    cb = function () {}
+  }
+  return rclient1.publish('test-pubsub', counter, (error) => {
+    if (error != null) {
+      console.error('[SENDING ERROR]', error.message)
+    }
+    if (error == null) {
+      counter += 1
+    }
+    return cb()
+  })
+}
 
 
-let previous = null;
-rclient2.subscribe("test-pubsub");
-rclient2.on("message", (channel, value) => {
-	value = parseInt(value, 10);
-	if ((value % 10) === 0) {
-		console.log(".");
-	}
-	if ((previous != null) && (value !== (previous + 1))) {
-		console.error("[RECEIVING ERROR]", `Counter not in order. Got ${value}, expected ${previous + 1}`);
-	}
-	return previous = value;
-});
+let previous = null
+rclient2.subscribe('test-pubsub')
+rclient2.on('message', (channel, value) => {
+  value = parseInt(value, 10)
+  if (value % 10 === 0) {
+    console.log('.')
+  }
+  if (previous != null && value !== previous + 1) {
+    console.error(
+      '[RECEIVING ERROR]',
+      `Counter not in order. Got ${value}, expected ${previous + 1}`
+    )
+  }
+  return (previous = value)
+})
 
 
-const PING_DELAY = 100;
-(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))();
+const PING_DELAY = 100
+;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()