github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-29 03:53:40 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #9723 from overleaf/bg-log-large-docs-in-docstore-archive

Browse source 
warn about oversized docs in docstore archiving

GitOrigin-RevId: 296a49b37b7f6733ef33f682ae40a8d3d478d9e8
This commit is contained in:
Brian Gough 2022-09-26 09:39:26 +01:00 committed by Copybot
parent 3e29e33685
commit 53d2074315
2 changed files with 17 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
services/docstore/app/js/DocArchiveManager.js
View file

@ -9,6 +9,9 @@ const RangeManager = require('./RangeManager')
const PersistorManager = require('./PersistorManager') const PersistorManager = require('./PersistorManager')
const pMap = require('p-map') const pMap = require('p-map')
const Bson = require('bson')
const BSON = new Bson()
const PARALLEL_JOBS = Settings.parallelArchiveJobs const PARALLEL_JOBS = Settings.parallelArchiveJobs
const UN_ARCHIVE_BATCH_SIZE = Settings.unArchiveBatchSize const UN_ARCHIVE_BATCH_SIZE = Settings.unArchiveBatchSize
@ -55,6 +58,19 @@ async function archiveDoc(projectId, docId) {
throw new Error('doc has no lines') throw new Error('doc has no lines')
} }
// warn about any oversized docs already in mongo
const linesSize = BSON.calculateObjectSize(doc.lines || {})
const rangesSize = BSON.calculateObjectSize(doc.ranges || {})
if (
linesSize > Settings.max_doc_length ||
rangesSize > Settings.max_doc_length
) {
logger.warn(
{ project_id: projectId, doc_id: doc._id, linesSize, rangesSize },
'large doc found when archiving project'
)
}
const json = JSON.stringify({ const json = JSON.stringify({
lines: doc.lines, lines: doc.lines,
ranges: doc.ranges, ranges: doc.ranges,

1
services/docstore/package.json
View file

@ -23,6 +23,7 @@
"@overleaf/settings": "^3.0.0", "@overleaf/settings": "^3.0.0",
"async": "^3.2.2", "async": "^3.2.2",
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"bson": "^1.1.4",
"bunyan": "^1.8.15", "bunyan": "^1.8.15",
"celebrate": "^13.0.4", "celebrate": "^13.0.4",
"express": "^4.17.1", "express": "^4.17.1",