github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Run format:fix

Browse source
This commit is contained in:
Alf Eaton 2021-05-25 14:27:45 +01:00
parent 03cbad4e9f
commit de50e59557
3 changed files with 173 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
services/document-updater/expire_docops.js
View file

@ -11,50 +11,68 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const Settings = require("settings-sharelatex");
const rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater);
let keys = Settings.redis.documentupdater.key_schema;
const async = require("async");
const RedisManager = require("./app/js/RedisManager");
const Settings = require('settings-sharelatex')
const rclient = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
let keys = Settings.redis.documentupdater.key_schema
const async = require('async')
const RedisManager = require('./app/js/RedisManager')
const getKeysFromNode = function(node, pattern, callback) {
let cursor = 0; // redis iterator
const keySet = {}; // use hash to avoid duplicate results
// scan over all keys looking for pattern
var doIteration = cb => node.scan(cursor, "MATCH", pattern, "COUNT", 1000, function(error, reply) {
if (error != null) { return callback(error); }
[cursor, keys] = Array.from(reply);
console.log("SCAN", keys.length);
for (const key of Array.from(keys)) {
keySet[key] = true;
}
if (cursor === '0') { // note redis returns string result not numeric
return callback(null, Object.keys(keySet));
} else {
return doIteration();
}
});
return doIteration();
};
const getKeysFromNode = function (node, pattern, callback) {
let cursor = 0 // redis iterator
const keySet = {} // use hash to avoid duplicate results
// scan over all keys looking for pattern
var doIteration = (cb) =>
node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) {
if (error != null) {
return callback(error)
}
;[cursor, keys] = Array.from(reply)
console.log('SCAN', keys.length)
for (const key of Array.from(keys)) {
keySet[key] = true
}
if (cursor === '0') {
// note redis returns string result not numeric
return callback(null, Object.keys(keySet))
} else {
return doIteration()
}
})
return doIteration()
}
const getKeys = function(pattern, callback) {
const nodes = (typeof rclient.nodes === 'function' ? rclient.nodes('master') : undefined) || [ rclient ];
console.log("GOT NODES", nodes.length);
const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb);
return async.concatSeries(nodes, doKeyLookupForNode, callback);
};
const getKeys = function (pattern, callback) {
const nodes = (typeof rclient.nodes === 'function'
? rclient.nodes('master')
: undefined) || [rclient]
console.log('GOT NODES', nodes.length)
const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb)
return async.concatSeries(nodes, doKeyLookupForNode, callback)
}
const TTL = 60 * 60; // 1 hour
const expireDocOps = callback => getKeys(keys.docOps({doc_id: "*"}), (error, keys) => async.mapSeries(keys,
function(key, cb) {
console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`);
return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb);
},
callback));
const TTL = 60 * 60 // 1 hour
const expireDocOps = (callback) =>
getKeys(keys.docOps({ doc_id: '*' }), (error, keys) =>
async.mapSeries(
keys,
function (key, cb) {
console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`)
return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb)
},
callback
)
)
setTimeout(() => // Give redis a chance to connect
expireDocOps(function(error) {
if (error != null) { throw error; }
return process.exit();
})
, 1000);
setTimeout(
() =>
// Give redis a chance to connect
expireDocOps(function (error) {
if (error != null) {
throw error
}
return process.exit()
}),
1000
)

104
services/document-updater/test/cluster_failover/coffee/test_blpop_failover.js
View file

@ -10,49 +10,69 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let listenInBackground, sendPings;
const redis = require("@overleaf/redis-wrapper");
const rclient1 = redis.createClient({cluster: [{
port: "7000",
host: "localhost"
}]});
const rclient2 = redis.createClient({cluster: [{
port: "7000",
host: "localhost"
}]});
let counter = 0;
const sendPing = function(cb) {
if (cb == null) { cb = function() {}; }
return rclient1.rpush("test-blpop", counter, (error) => {
if (error != null) { console.error("[SENDING ERROR]", error.message); }
if ((error == null)) {
counter += 1;
}
return cb();
});
};
let previous = null;
const listenForPing = cb => rclient2.blpop("test-blpop", 200, (error, result) => {
if (error != null) { return cb(error); }
let [key, value] = Array.from(result);
value = parseInt(value, 10);
if ((value % 10) === 0) {
console.log(".");
let listenInBackground, sendPings
const redis = require('@overleaf/redis-wrapper')
const rclient1 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
if ((previous != null) && (value !== (previous + 1))) {
error = new Error(`Counter not in order. Got ${value}, expected ${previous + 1}`);
]
})
const rclient2 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
previous = value;
return cb(error, value);
});
]
})
const PING_DELAY = 100;
(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))();
let counter = 0
const sendPing = function (cb) {
if (cb == null) {
cb = function () {}
}
return rclient1.rpush('test-blpop', counter, (error) => {
if (error != null) {
console.error('[SENDING ERROR]', error.message)
}
if (error == null) {
counter += 1
}
return cb()
})
}
(listenInBackground = () => listenForPing((error, value) => {
if (error) { console.error("[RECEIVING ERROR]", error.message); }
return setTimeout(listenInBackground);
}))();
let previous = null
const listenForPing = (cb) =>
rclient2.blpop('test-blpop', 200, (error, result) => {
if (error != null) {
return cb(error)
}
let [key, value] = Array.from(result)
value = parseInt(value, 10)
if (value % 10 === 0) {
console.log('.')
}
if (previous != null && value !== previous + 1) {
error = new Error(
`Counter not in order. Got ${value}, expected ${previous + 1}`
)
}
previous = value
return cb(error, value)
})
const PING_DELAY = 100
;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()
;(listenInBackground = () =>
listenForPing((error, value) => {
if (error) {
console.error('[RECEIVING ERROR]', error.message)
}
return setTimeout(listenInBackground)
}))()

85
services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.js
View file

@ -9,42 +9,57 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let sendPings;
const redis = require("@overleaf/redis-wrapper");
const rclient1 = redis.createClient({cluster: [{
port: "7000",
host: "localhost"
}]});
let sendPings
const redis = require('@overleaf/redis-wrapper')
const rclient1 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
]
})
const rclient2 = redis.createClient({cluster: [{
port: "7000",
host: "localhost"
}]});
const rclient2 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
]
})
let counter = 0;
const sendPing = function(cb) {
if (cb == null) { cb = function() {}; }
return rclient1.publish("test-pubsub", counter, (error) => {
if (error != null) { console.error("[SENDING ERROR]", error.message); }
if ((error == null)) {
counter += 1;
}
return cb();
});
};
let counter = 0
const sendPing = function (cb) {
if (cb == null) {
cb = function () {}
}
return rclient1.publish('test-pubsub', counter, (error) => {
if (error != null) {
console.error('[SENDING ERROR]', error.message)
}
if (error == null) {
counter += 1
}
return cb()
})
}
let previous = null;
rclient2.subscribe("test-pubsub");
rclient2.on("message", (channel, value) => {
value = parseInt(value, 10);
if ((value % 10) === 0) {
console.log(".");
}
if ((previous != null) && (value !== (previous + 1))) {
console.error("[RECEIVING ERROR]", `Counter not in order. Got ${value}, expected ${previous + 1}`);
}
return previous = value;
});
let previous = null
rclient2.subscribe('test-pubsub')
rclient2.on('message', (channel, value) => {
value = parseInt(value, 10)
if (value % 10 === 0) {
console.log('.')
}
if (previous != null && value !== previous + 1) {
console.error(
'[RECEIVING ERROR]',
`Counter not in order. Got ${value}, expected ${previous + 1}`
)
}
return (previous = value)
})
const PING_DELAY = 100;
(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))();
const PING_DELAY = 100
;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()