diff --git a/services/document-updater/expire_docops.js b/services/document-updater/expire_docops.js index 811b0cc019..ffafbe7255 100644 --- a/services/document-updater/expire_docops.js +++ b/services/document-updater/expire_docops.js @@ -11,50 +11,68 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const Settings = require("settings-sharelatex"); -const rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater); -let keys = Settings.redis.documentupdater.key_schema; -const async = require("async"); -const RedisManager = require("./app/js/RedisManager"); +const Settings = require('settings-sharelatex') +const rclient = require('@overleaf/redis-wrapper').createClient( + Settings.redis.documentupdater +) +let keys = Settings.redis.documentupdater.key_schema +const async = require('async') +const RedisManager = require('./app/js/RedisManager') -const getKeysFromNode = function(node, pattern, callback) { - let cursor = 0; // redis iterator - const keySet = {}; // use hash to avoid duplicate results - // scan over all keys looking for pattern - var doIteration = cb => node.scan(cursor, "MATCH", pattern, "COUNT", 1000, function(error, reply) { - if (error != null) { return callback(error); } - [cursor, keys] = Array.from(reply); - console.log("SCAN", keys.length); - for (const key of Array.from(keys)) { - keySet[key] = true; - } - if (cursor === '0') { // note redis returns string result not numeric - return callback(null, Object.keys(keySet)); - } else { - return doIteration(); - } - }); - return doIteration(); -}; +const getKeysFromNode = function (node, pattern, callback) { + let cursor = 0 // redis iterator + const keySet = {} // use hash to avoid duplicate results + // scan over all keys looking for pattern + var doIteration = (cb) => + node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) { + if (error != null) { + return callback(error) + } + ;[cursor, keys] = Array.from(reply) + console.log('SCAN', keys.length) + for (const key of Array.from(keys)) { + keySet[key] = true + } + if (cursor === '0') { + // note redis returns string result not numeric + return callback(null, Object.keys(keySet)) + } else { + return doIteration() + } + }) + return doIteration() +} -const getKeys = function(pattern, callback) { - const nodes = (typeof rclient.nodes === 'function' ? rclient.nodes('master') : undefined) || [ rclient ]; - console.log("GOT NODES", nodes.length); - const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb); - return async.concatSeries(nodes, doKeyLookupForNode, callback); -}; +const getKeys = function (pattern, callback) { + const nodes = (typeof rclient.nodes === 'function' + ? rclient.nodes('master') + : undefined) || [rclient] + console.log('GOT NODES', nodes.length) + const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb) + return async.concatSeries(nodes, doKeyLookupForNode, callback) +} -const TTL = 60 * 60; // 1 hour -const expireDocOps = callback => getKeys(keys.docOps({doc_id: "*"}), (error, keys) => async.mapSeries(keys, - function(key, cb) { - console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`); - return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb); - }, - callback)); +const TTL = 60 * 60 // 1 hour +const expireDocOps = (callback) => + getKeys(keys.docOps({ doc_id: '*' }), (error, keys) => + async.mapSeries( + keys, + function (key, cb) { + console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`) + return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb) + }, + callback + ) + ) -setTimeout(() => // Give redis a chance to connect -expireDocOps(function(error) { - if (error != null) { throw error; } - return process.exit(); -}) -, 1000); +setTimeout( + () => + // Give redis a chance to connect + expireDocOps(function (error) { + if (error != null) { + throw error + } + return process.exit() + }), + 1000 +) diff --git a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js index a356e683d4..3d7cae99da 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js @@ -10,49 +10,69 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let listenInBackground, sendPings; -const redis = require("@overleaf/redis-wrapper"); -const rclient1 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); - -const rclient2 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); - -let counter = 0; -const sendPing = function(cb) { - if (cb == null) { cb = function() {}; } - return rclient1.rpush("test-blpop", counter, (error) => { - if (error != null) { console.error("[SENDING ERROR]", error.message); } - if ((error == null)) { - counter += 1; - } - return cb(); - }); -}; - -let previous = null; -const listenForPing = cb => rclient2.blpop("test-blpop", 200, (error, result) => { - if (error != null) { return cb(error); } - let [key, value] = Array.from(result); - value = parseInt(value, 10); - if ((value % 10) === 0) { - console.log("."); +let listenInBackground, sendPings +const redis = require('@overleaf/redis-wrapper') +const rclient1 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' } - if ((previous != null) && (value !== (previous + 1))) { - error = new Error(`Counter not in order. Got ${value}, expected ${previous + 1}`); + ] +}) + +const rclient2 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' } - previous = value; - return cb(error, value); -}); + ] +}) -const PING_DELAY = 100; -(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); +let counter = 0 +const sendPing = function (cb) { + if (cb == null) { + cb = function () {} + } + return rclient1.rpush('test-blpop', counter, (error) => { + if (error != null) { + console.error('[SENDING ERROR]', error.message) + } + if (error == null) { + counter += 1 + } + return cb() + }) +} -(listenInBackground = () => listenForPing((error, value) => { - if (error) { console.error("[RECEIVING ERROR]", error.message); } - return setTimeout(listenInBackground); -}))(); +let previous = null +const listenForPing = (cb) => + rclient2.blpop('test-blpop', 200, (error, result) => { + if (error != null) { + return cb(error) + } + let [key, value] = Array.from(result) + value = parseInt(value, 10) + if (value % 10 === 0) { + console.log('.') + } + if (previous != null && value !== previous + 1) { + error = new Error( + `Counter not in order. Got ${value}, expected ${previous + 1}` + ) + } + previous = value + return cb(error, value) + }) + +const PING_DELAY = 100 +;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() + +;(listenInBackground = () => + listenForPing((error, value) => { + if (error) { + console.error('[RECEIVING ERROR]', error.message) + } + return setTimeout(listenInBackground) + }))() diff --git a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js index 670c7afa3a..627fb82a0d 100644 --- a/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js +++ b/services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js @@ -9,42 +9,57 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let sendPings; -const redis = require("@overleaf/redis-wrapper"); -const rclient1 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); +let sendPings +const redis = require('@overleaf/redis-wrapper') +const rclient1 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) -const rclient2 = redis.createClient({cluster: [{ - port: "7000", - host: "localhost" -}]}); +const rclient2 = redis.createClient({ + cluster: [ + { + port: '7000', + host: 'localhost' + } + ] +}) -let counter = 0; -const sendPing = function(cb) { - if (cb == null) { cb = function() {}; } - return rclient1.publish("test-pubsub", counter, (error) => { - if (error != null) { console.error("[SENDING ERROR]", error.message); } - if ((error == null)) { - counter += 1; - } - return cb(); - }); -}; +let counter = 0 +const sendPing = function (cb) { + if (cb == null) { + cb = function () {} + } + return rclient1.publish('test-pubsub', counter, (error) => { + if (error != null) { + console.error('[SENDING ERROR]', error.message) + } + if (error == null) { + counter += 1 + } + return cb() + }) +} -let previous = null; -rclient2.subscribe("test-pubsub"); -rclient2.on("message", (channel, value) => { - value = parseInt(value, 10); - if ((value % 10) === 0) { - console.log("."); - } - if ((previous != null) && (value !== (previous + 1))) { - console.error("[RECEIVING ERROR]", `Counter not in order. Got ${value}, expected ${previous + 1}`); - } - return previous = value; -}); +let previous = null +rclient2.subscribe('test-pubsub') +rclient2.on('message', (channel, value) => { + value = parseInt(value, 10) + if (value % 10 === 0) { + console.log('.') + } + if (previous != null && value !== previous + 1) { + console.error( + '[RECEIVING ERROR]', + `Counter not in order. Got ${value}, expected ${previous + 1}` + ) + } + return (previous = value) +}) -const PING_DELAY = 100; -(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))(); +const PING_DELAY = 100 +;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()