github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #177 from overleaf/ae-decaffeinate-config

Browse source 
Decaffeinate scripts
This commit is contained in:
Alf Eaton 2021-05-28 12:25:44 +01:00 committed by GitHub
commit d54424bd52
7 changed files with 185 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
services/document-updater/expire_docops.coffee
View file

@ -1,44 +0,0 @@
Settings = require "settings-sharelatex"
rclient = require("@overleaf/redis-wrapper").createClient(Settings.redis.documentupdater)
keys = Settings.redis.documentupdater.key_schema
async = require "async"
RedisManager = require "./app/js/RedisManager"
getKeysFromNode = (node, pattern, callback) ->
cursor = 0 # redis iterator
keySet = {} # use hash to avoid duplicate results
# scan over all keys looking for pattern
doIteration = (cb) ->
node.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) ->
return callback(error) if error?
[cursor, keys] = reply
console.log "SCAN", keys.length
for key in keys
keySet[key] = true
if cursor == '0' # note redis returns string result not numeric
return callback(null, Object.keys(keySet))
else
doIteration()
doIteration()
getKeys = (pattern, callback) ->
nodes = rclient.nodes?('master') || [ rclient ];
console.log "GOT NODES", nodes.length
doKeyLookupForNode = (node, cb) ->
getKeysFromNode node, pattern, cb
async.concatSeries nodes, doKeyLookupForNode, callback
TTL = 60 * 60 # 1 hour
expireDocOps = (callback) ->
getKeys keys.docOps(doc_id: "*"), (error, keys) ->
async.mapSeries keys,
(key, cb) ->
console.log "EXPIRE #{key} #{RedisManager.DOC_OPS_TTL}"
rclient.expire key, RedisManager.DOC_OPS_TTL, cb
callback
setTimeout () -> # Give redis a chance to connect
expireDocOps (error) ->
throw error if error?
process.exit()
, 1000

65
services/document-updater/expire_docops.js Normal file
View file

@ -0,0 +1,65 @@
const Settings = require('settings-sharelatex')
const rclient = require('@overleaf/redis-wrapper').createClient(
Settings.redis.documentupdater
)
let keys = Settings.redis.documentupdater.key_schema
const async = require('async')
const RedisManager = require('./app/js/RedisManager')
const getKeysFromNode = function (node, pattern, callback) {
let cursor = 0 // redis iterator
const keySet = {} // use hash to avoid duplicate results
// scan over all keys looking for pattern
const doIteration = () =>
node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) {
if (error) {
return callback(error)
}
;[cursor, keys] = reply
console.log('SCAN', keys.length)
for (const key of keys) {
keySet[key] = true
}
if (cursor === '0') {
// note redis returns string result not numeric
return callback(null, Object.keys(keySet))
} else {
return doIteration()
}
})
return doIteration()
}
const getKeys = function (pattern, callback) {
const nodes = (typeof rclient.nodes === 'function'
? rclient.nodes('master')
: undefined) || [rclient]
console.log('GOT NODES', nodes.length)
const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb)
return async.concatSeries(nodes, doKeyLookupForNode, callback)
}
const expireDocOps = (callback) =>
// eslint-disable-next-line handle-callback-err
getKeys(keys.docOps({ doc_id: '*' }), (error, keys) =>
async.mapSeries(
keys,
function (key, cb) {
console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`)
return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb)
},
callback
)
)
setTimeout(
() =>
// Give redis a chance to connect
expireDocOps(function (error) {
if (error) {
throw error
}
return process.exit()
}),
1000
)

41
services/document-updater/test/cluster_failover/coffee/test_blpop_failover.coffee
View file

@ -1,41 +0,0 @@
redis = require "@overleaf/redis-wrapper"
rclient1 = redis.createClient(cluster: [{
port: "7000"
host: "localhost"
}])
rclient2 = redis.createClient(cluster: [{
port: "7000"
host: "localhost"
}])
counter = 0
sendPing = (cb = () ->) ->
rclient1.rpush "test-blpop", counter, (error) ->
console.error "[SENDING ERROR]", error.message if error?
if !error?
counter += 1
cb()
previous = null
listenForPing = (cb) ->
rclient2.blpop "test-blpop", 200, (error, result) ->
return cb(error) if error?
[key, value] = result
value = parseInt(value, 10)
if value % 10 == 0
console.log "."
if previous? and value != previous + 1
error = new Error("Counter not in order. Got #{value}, expected #{previous + 1}")
previous = value
return cb(error, value)
PING_DELAY = 100
do sendPings = () ->
sendPing () ->
setTimeout sendPings, PING_DELAY
do listenInBackground = (cb = () ->) ->
listenForPing (error, value) ->
console.error "[RECEIVING ERROR]", error.message if error
setTimeout listenInBackground

33
services/document-updater/test/cluster_failover/coffee/test_pubsub_failover.coffee
View file

@ -1,33 +0,0 @@
redis = require "@overleaf/redis-wrapper"
rclient1 = redis.createClient(cluster: [{
port: "7000"
host: "localhost"
}])
rclient2 = redis.createClient(cluster: [{
port: "7000"
host: "localhost"
}])
counter = 0
sendPing = (cb = () ->) ->
rclient1.publish "test-pubsub", counter, (error) ->
console.error "[SENDING ERROR]", error.message if error?
if !error?
counter += 1
cb()
previous = null
rclient2.subscribe "test-pubsub"
rclient2.on "message", (channel, value) ->
value = parseInt(value, 10)
if value % 10 == 0
console.log "."
if previous? and value != previous + 1
console.error "[RECEIVING ERROR]", "Counter not in order. Got #{value}, expected #{previous + 1}"
previous = value
PING_DELAY = 100
do sendPings = () ->
sendPing () ->
setTimeout sendPings, PING_DELAY

65
services/document-updater/test/cluster_failover/js/test_blpop_failover.js Normal file
View file

@ -0,0 +1,65 @@
let listenInBackground, sendPings
const redis = require('@overleaf/redis-wrapper')
const rclient1 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
]
})
const rclient2 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
]
})
let counter = 0
const sendPing = function (cb) {
if (cb == null) {
cb = function () {}
}
return rclient1.rpush('test-blpop', counter, (error) => {
if (error != null) {
console.error('[SENDING ERROR]', error.message)
}
if (error == null) {
counter += 1
}
return cb()
})
}
let previous = null
const listenForPing = (cb) =>
rclient2.blpop('test-blpop', 200, (error, result) => {
if (error != null) {
return cb(error)
}
let [, value] = Array.from(result)
value = parseInt(value, 10)
if (value % 10 === 0) {
console.log('.')
}
if (previous != null && value !== previous + 1) {
error = new Error(
`Counter not in order. Got ${value}, expected ${previous + 1}`
)
}
previous = value
return cb(error, value)
})
const PING_DELAY = 100
;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()
;(listenInBackground = () =>
listenForPing((error) => {
if (error) {
console.error('[RECEIVING ERROR]', error.message)
}
return setTimeout(listenInBackground)
}))()

54
services/document-updater/test/cluster_failover/js/test_pubsub_failover.js Normal file
View file

@ -0,0 +1,54 @@
let sendPings
const redis = require('@overleaf/redis-wrapper')
const rclient1 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
]
})
const rclient2 = redis.createClient({
cluster: [
{
port: '7000',
host: 'localhost'
}
]
})
let counter = 0
const sendPing = function (cb) {
if (cb == null) {
cb = function () {}
}
return rclient1.publish('test-pubsub', counter, (error) => {
if (error) {
console.error('[SENDING ERROR]', error.message)
}
if (error == null) {
counter += 1
}
return cb()
})
}
let previous = null
rclient2.subscribe('test-pubsub')
rclient2.on('message', (channel, value) => {
value = parseInt(value, 10)
if (value % 10 === 0) {
console.log('.')
}
if (previous != null && value !== previous + 1) {
console.error(
'[RECEIVING ERROR]',
`Counter not in order. Got ${value}, expected ${previous + 1}`
)
}
return (previous = value)
})
const PING_DELAY = 100
;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))()

2
services/document-updater/test/stress/js/run.js
View file

@ -16,7 +16,7 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const DocUpdaterClient = require('../../acceptance/coffee/helpers/DocUpdaterClient')
const DocUpdaterClient = require('../../acceptance/js/helpers/DocUpdaterClient')
// MockTrackChangesApi = require "../../acceptance/js/helpers/MockTrackChangesApi"
// MockWebApi = require "../../acceptance/js/helpers/MockWebApi"
const assert = require('assert')