github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-17 10:58:49 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request from overleaf/em-project-imports

Browse source 
Decaf cleanup

GitOrigin-RevId: 6ee288b8c8572d799032650580e2e679d856ee09
This commit is contained in:
Timothée Alby 2019-11-04 16:50:15 +07:00 • committed by sharelatex
parent 56566106c2
commit 12dd42724c
4 changed files with 734 additions and 895 deletions
services/web
app/src
Features/Project
ProjectEntityMongoUpdateHandler.jsProjectLocator.js
infrastructure
LockManager.js
test/unit/src/Project
ProjectEntityMongoUpdateHandlerTests.js

751
services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js
View file

File diff suppressed because it is too large Load diff

196
services/web/app/src/Features/Project/ProjectLocator.js
View file

@ -1,25 +1,3 @@
/* eslint-disable
camelcase,
handle-callback-err,
max-len,
no-array-constructor,
no-return-assign,
no-undef,
no-unused-vars,
standard/no-callback-literal,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS203: Remove `|| {}` from converted for-own loops
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ProjectLocator
const { Project } = require('../../models/Project')
const ProjectGetter = require('./ProjectGetter')
const ProjectHelper = require('./ProjectHelper')
const Errors = require('../Errors/Errors')
@ -27,92 +5,93 @@ const _ = require('underscore')
const logger = require('logger-sharelatex')
const async = require('async')
module.exports = ProjectLocator = {
const ProjectLocator = {
findElement(options, _callback) {
if (_callback == null) {
_callback = function(err, element, path, parentFolder) {}
}
const callback = function(...args) {
_callback(...Array.from(args || []))
return (_callback = function() {})
}
// The search algorithm below potentially invokes the callback multiple
// times.
const callback = _.once(_callback)
const { project, project_id, element_id, type } = options
const {
project,
project_id: projectId,
element_id: elementId,
type
} = options
const elementType = sanitizeTypeOfElement(type)
let count = 0
const endOfBranch = function() {
if (--count === 0) {
logger.warn(
`element ${element_id} could not be found for project ${project_id ||
`element ${elementId} could not be found for project ${projectId ||
project._id}`
)
return callback(new Errors.NotFoundError('entity not found'))
callback(new Errors.NotFoundError('entity not found'))
}
}
var search = function(searchFolder, path) {
function search(searchFolder, path) {
count++
const element = _.find(
searchFolder[elementType],
el => (el != null ? el._id : undefined) + '' === element_id + ''
el => (el != null ? el._id : undefined) + '' === elementId + ''
) // need to ToString both id's for robustness
if (
element == null &&
searchFolder.folders != null &&
searchFolder.folders.length !== 0
) {
_.each(searchFolder.folders, function(folder, index) {
_.each(searchFolder.folders, (folder, index) => {
if (folder == null) {
return
}
const newPath = {}
for (let key of Object.keys(path || {})) {
for (let key of Object.keys(path)) {
const value = path[key]
newPath[key] = value
} // make a value copy of the string
newPath.fileSystem += `/${folder.name}`
newPath.mongo += `.folders.${index}`
return search(folder, newPath)
search(folder, newPath)
})
endOfBranch()
} else if (element != null) {
const elementPlaceInArray = getIndexOf(
searchFolder[elementType],
element_id
elementId
)
path.fileSystem += `/${element.name}`
path.mongo += `.${elementType}.${elementPlaceInArray}`
return callback(null, element, path, searchFolder)
callback(null, element, path, searchFolder)
} else if (element == null) {
return endOfBranch()
endOfBranch()
}
}
const path = { fileSystem: '', mongo: 'rootFolder.0' }
const startSearch = function(project) {
if (element_id + '' === project.rootFolder[0]._id + '') {
return callback(null, project.rootFolder[0], path, null)
const startSearch = project => {
if (elementId + '' === project.rootFolder[0]._id + '') {
callback(null, project.rootFolder[0], path, null)
} else {
return search(project.rootFolder[0], path)
search(project.rootFolder[0], path)
}
}
if (project != null) {
return startSearch(project)
startSearch(project)
} else {
return ProjectGetter.getProject(
project_id,
ProjectGetter.getProject(
projectId,
{ rootFolder: true, rootDoc_id: true },
function(err, project) {
(err, project) => {
if (err != null) {
return callback(err)
}
if (project == null) {
return callback(new Errors.NotFoundError('project not found'))
}
return startSearch(project)
startSearch(project)
}
)
}
@ -121,9 +100,9 @@ module.exports = ProjectLocator = {
findRootDoc(opts, callback) {
const getRootDoc = project => {
if (project.rootDoc_id != null) {
return this.findElement(
this.findElement(
{ project, element_id: project.rootDoc_id, type: 'docs' },
function(error, ...args) {
(error, ...args) => {
if (error != null) {
if (error instanceof Errors.NotFoundError) {
return callback(null, null)
@ -131,26 +110,26 @@ module.exports = ProjectLocator = {
return callback(error)
}
}
return callback(null, ...Array.from(args))
callback(null, ...args)
}
)
} else {
return callback(null, null)
callback(null, null)
}
}
const { project, project_id } = opts
const { project, project_id: projectId } = opts
if (project != null) {
return getRootDoc(project)
getRootDoc(project)
} else {
return ProjectGetter.getProject(
project_id,
ProjectGetter.getProject(
projectId,
{ rootFolder: true, rootDoc_id: true },
function(err, project) {
(err, project) => {
if (err != null) {
logger.warn({ err }, 'error getting project')
return callback(err)
callback(err)
} else {
return getRootDoc(project)
getRootDoc(project)
}
}
)
@ -158,30 +137,27 @@ module.exports = ProjectLocator = {
},
findElementByPath(options, callback) {
if (callback == null) {
callback = function(err, foundEntity, type) {}
}
const { project, project_id, path, exactCaseMatch } = options
const { project, project_id: projectId, path, exactCaseMatch } = options
if (path == null) {
return new Error('no path provided for findElementByPath')
}
if (project != null) {
return ProjectLocator._findElementByPathWithProject(
ProjectLocator._findElementByPathWithProject(
project,
path,
exactCaseMatch,
callback
)
} else {
return ProjectGetter.getProject(
project_id,
ProjectGetter.getProject(
projectId,
{ rootFolder: true, rootDoc_id: true },
function(err, project) {
(err, project) => {
if (err != null) {
return callback(err)
}
return ProjectLocator._findElementByPathWithProject(
ProjectLocator._findElementByPathWithProject(
project,
path,
exactCaseMatch,
@ -194,9 +170,6 @@ module.exports = ProjectLocator = {
_findElementByPathWithProject(project, needlePath, exactCaseMatch, callback) {
let matchFn
if (callback == null) {
callback = function(err, foundEntity, type) {}
}
if (exactCaseMatch) {
matchFn = (a, b) => a === b
} else {
@ -205,13 +178,13 @@ module.exports = ProjectLocator = {
(b != null ? b.toLowerCase() : undefined)
}
var getParentFolder = function(haystackFolder, foldersList, level, cb) {
function getParentFolder(haystackFolder, foldersList, level, cb) {
if (foldersList.length === 0) {
return cb(null, haystackFolder)
}
const needleFolderName = foldersList[level]
let found = false
for (let folder of Array.from(haystackFolder.folders)) {
for (let folder of haystackFolder.folders) {
if (matchFn(folder.name, needleFolderName)) {
found = true
if (level === foldersList.length - 1) {
@ -222,34 +195,36 @@ module.exports = ProjectLocator = {
}
}
if (!found) {
return cb(
`not found project: ${
project._id
} search path: ${needlePath}, folder ${
foldersList[level]
} could not be found`
cb(
new Error(
`not found project: ${
project._id
} search path: ${needlePath}, folder ${
foldersList[level]
} could not be found`
)
)
}
}
const getEntity = function(folder, entityName, cb) {
function getEntity(folder, entityName, cb) {
let result, type
if (entityName == null) {
return cb(null, folder, 'folder')
}
for (let file of Array.from(folder.fileRefs || [])) {
for (let file of folder.fileRefs || []) {
if (matchFn(file != null ? file.name : undefined, entityName)) {
result = file
type = 'file'
}
}
for (let doc of Array.from(folder.docs || [])) {
for (let doc of folder.docs || []) {
if (matchFn(doc != null ? doc.name : undefined, entityName)) {
result = doc
type = 'doc'
}
}
for (let childFolder of Array.from(folder.folders || [])) {
for (let childFolder of folder.folders || []) {
if (
matchFn(
childFolder != null ? childFolder.name : undefined,
@ -262,25 +237,20 @@ module.exports = ProjectLocator = {
}
if (result != null) {
return cb(null, result, type)
cb(null, result, type)
} else {
return cb(
`not found project: ${
project._id
} search path: ${needlePath}, entity ${entityName} could not be found`
cb(
new Error(
`not found project: ${
project._id
} search path: ${needlePath}, entity ${entityName} could not be found`
)
)
}
}
if (typeof err !== 'undefined' && err !== null) {
logger.warn(
{ err, project_id: project._id },
'error getting project for finding element'
)
return callback(err)
}
if (project == null) {
return callback('Tried to find an element for a null project')
return callback(new Error('Tried to find an element for a null project'))
}
if (needlePath === '' || needlePath === '/') {
return callback(null, project.rootFolder[0], 'folder')
@ -294,22 +264,22 @@ module.exports = ProjectLocator = {
const rootFolder = project.rootFolder[0]
logger.log(
{ project_id: project._id, path: needlePath, foldersList },
{ projectId: project._id, path: needlePath, foldersList },
'looking for element by path'
)
const jobs = new Array()
const jobs = []
jobs.push(cb => getParentFolder(rootFolder, foldersList, 0, cb))
jobs.push((folder, cb) => getEntity(folder, needleName, cb))
return async.waterfall(jobs, callback)
async.waterfall(jobs, callback)
},
findUsersProjectByName(user_id, projectName, callback) {
return ProjectGetter.findAllUsersProjects(
user_id,
findUsersProjectByName(userId, projectName, callback) {
ProjectGetter.findAllUsersProjects(
userId,
'name archived trashed',
function(err, allProjects) {
if (typeof error !== 'undefined' && error !== null) {
return callback(error)
(err, allProjects) => {
if (err != null) {
return callback(err)
}
const { owned, readAndWrite } = allProjects
const projects = owned.concat(readAndWrite)
@ -318,19 +288,19 @@ module.exports = ProjectLocator = {
projects,
project =>
project.name.toLowerCase() === projectName &&
!ProjectHelper.isArchivedOrTrashed(project, user_id)
!ProjectHelper.isArchivedOrTrashed(project, userId)
)
logger.log(
{ user_id, projectName, totalProjects: projects.length, project },
{ userId, projectName, totalProjects: projects.length, project },
'looking for project by name'
)
return callback(null, project)
callback(null, project)
}
)
}
}
var sanitizeTypeOfElement = function(elementType) {
function sanitizeTypeOfElement(elementType) {
const lastChar = elementType.slice(-1)
if (lastChar !== 's') {
elementType += 's'
@ -341,7 +311,7 @@ var sanitizeTypeOfElement = function(elementType) {
return elementType
}
var getIndexOf = function(searchEntity, id) {
function getIndexOf(searchEntity, id) {
const { length } = searchEntity
let count = 0
while (count < length) {
@ -355,3 +325,5 @@ var getIndexOf = function(searchEntity, id) {
count++
}
}
module.exports = ProjectLocator

87
services/web/app/src/infrastructure/LockManager.js
View file

@ -1,20 +1,4 @@
/* eslint-disable
handle-callback-err,
max-len,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LockManager
const metrics = require('metrics-sharelatex')
const Settings = require('settings-sharelatex')
const RedisWrapper = require('./RedisWrapper')
const rclient = RedisWrapper.client('lock')
const logger = require('logger-sharelatex')
@ -29,7 +13,7 @@ let COUNT = 0
const LOCK_QUEUES = new Map() // queue lock requests for each name/id so they get the lock on a first-come first-served basis
module.exports = LockManager = {
const LockManager = {
LOCK_TEST_INTERVAL: 50, // 50ms between each test of the lock
MAX_TEST_INTERVAL: 1000, // back off to 1s between each test of the lock
MAX_LOCK_WAIT_TIME: 10000, // 10s maximum time to spend trying to get the lock
@ -51,23 +35,20 @@ module.exports = LockManager = {
// runner must be a function accepting a callback, e.g. runner = (cb) ->
// This error is defined here so we get a useful stacktrace
if (callback == null) {
callback = function(error) {}
}
const slowExecutionError = new Error('slow execution during lock')
const timer = new metrics.Timer(`lock.${namespace}`)
const key = `lock:web:${namespace}:${id}`
LockManager._getLock(key, namespace, function(error, lockValue) {
LockManager._getLock(key, namespace, (error, lockValue) => {
if (error != null) {
return callback(error)
}
// The lock can expire in redis but the process carry on. This setTimout call
// The lock can expire in redis but the process carry on. This setTimeout call
// is designed to log if this happens.
const countIfExceededLockTimeout = function() {
function countIfExceededLockTimeout() {
metrics.inc(`lock.${namespace}.exceeded_lock_timeout`)
return logger.log('exceeded lock timeout', {
logger.log('exceeded lock timeout', {
namespace,
id,
slowExecutionError
@ -78,8 +59,8 @@ module.exports = LockManager = {
LockManager.REDIS_LOCK_EXPIRY * 1000
)
return runner((error1, ...values) =>
LockManager._releaseLock(key, lockValue, function(error2) {
runner((error1, ...values) =>
LockManager._releaseLock(key, lockValue, error2 => {
clearTimeout(exceededLockTimeout)
const timeTaken = new Date() - timer.start
@ -97,16 +78,13 @@ module.exports = LockManager = {
if (error != null) {
return callback(error)
}
return callback(null, ...Array.from(values))
callback(null, ...values)
})
)
})
},
_tryLock(key, namespace, callback) {
if (callback == null) {
callback = function(err, isFree, lockValue) {}
}
const lockValue = LockManager.randomLock()
rclient.set(
key,
@ -114,17 +92,17 @@ module.exports = LockManager = {
'EX',
LockManager.REDIS_LOCK_EXPIRY,
'NX',
function(err, gotLock) {
(err, gotLock) => {
if (err != null) {
return callback(err)
}
if (gotLock === 'OK') {
metrics.inc(`lock.${namespace}.try.success`)
return callback(err, true, lockValue)
callback(err, true, lockValue)
} else {
metrics.inc(`lock.${namespace}.try.failed`)
logger.log({ key, redis_response: gotLock }, 'lock is locked')
return callback(err, false)
callback(err, false)
}
}
)
@ -133,15 +111,12 @@ module.exports = LockManager = {
// it's sufficient to serialize within a process because that is where the parallel operations occur
_getLock(key, namespace, callback) {
// this is what we need to do for each lock we want to request
if (callback == null) {
callback = function(error, lockValue) {}
}
const task = next =>
LockManager._getLockByPolling(key, namespace, function(error, lockValue) {
LockManager._getLockByPolling(key, namespace, (error, lockValue) => {
// tell the queue to start trying to get the next lock (if any)
next()
// we have got a lock result, so we can continue with our own execution
return callback(error, lockValue)
callback(error, lockValue)
})
// create a queue for this key if needed
const queueName = `${key}:${namespace}`
@ -154,53 +129,43 @@ module.exports = LockManager = {
// remove the queue object when queue is empty
queue.drain = () => LOCK_QUEUES.delete(queueName)
// store the queue in our global map
return LOCK_QUEUES.set(queueName, queue)
LOCK_QUEUES.set(queueName, queue)
} else {
// queue the request to get the lock
return queue.push(task)
queue.push(task)
}
},
_getLockByPolling(key, namespace, callback) {
let attempt
if (callback == null) {
callback = function(error, lockValue) {}
}
const startTime = Date.now()
const testInterval = LockManager.LOCK_TEST_INTERVAL
let attempts = 0
return (attempt = function() {
function attempt() {
if (Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME) {
metrics.inc(`lock.${namespace}.get.failed`)
return callback(new Error('Timeout'))
}
attempts += 1
return LockManager._tryLock(key, namespace, function(
error,
gotLock,
lockValue
) {
LockManager._tryLock(key, namespace, (error, gotLock, lockValue) => {
if (error != null) {
return callback(error)
}
if (gotLock) {
metrics.gauge(`lock.${namespace}.get.success.tries`, attempts)
return callback(null, lockValue)
callback(null, lockValue)
} else {
return setTimeout(attempt, testInterval)
setTimeout(attempt, testInterval)
}
})
})()
}
attempt()
},
_releaseLock(key, lockValue, callback) {
rclient.eval(LockManager.unlockScript, 1, key, lockValue, function(
err,
result
) {
rclient.eval(LockManager.unlockScript, 1, key, lockValue, (err, result) => {
if (err != null) {
return callback(err)
callback(err)
} else if (result != null && result !== 1) {
// successful unlock should release exactly one key
logger.warn(
@ -208,10 +173,12 @@ module.exports = LockManager = {
'unlocking error'
)
metrics.inc('unlock-error')
return callback(new Error('tried to release timed out lock'))
callback(new Error('tried to release timed out lock'))
} else {
return callback(null, result)
callback(null, result)
}
})
}
}
module.exports = LockManager

595
services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js
View file

File diff suppressed because it is too large Load diff