mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #15382 from overleaf/csh-issue-11625-mongo-ug-5-history-v1
Upgrade mongodb module for history-v1 from 4.11.0 to 6.2.0 GitOrigin-RevId: 2f0a8c07f2797ac23d202858613715b177b41e15
This commit is contained in:
Christopher Hoskin
Copybot
parent
7da8e3b646
commit
0ae174a47b
7 changed files with 102 additions and 89 deletions
145
package-lock.json
generated
145
package-lock.json
generated
|
|
@ -38426,7 +38426,7 @@
|
|||
"jsonwebtoken": "^9.0.0",
|
||||
"knex": "^2.4.0",
|
||||
"lodash": "^4.17.19",
|
||||
"mongodb": "^4.11.0",
|
||||
"mongodb": "^6.2.0",
|
||||
"overleaf-editor-core": "*",
|
||||
"pg": "^8.7.1",
|
||||
"swagger-tools": "^0.10.4",
|
||||
|
|
@ -38458,37 +38458,11 @@
|
|||
}
|
||||
},
|
||||
"services/history-v1/node_modules/bson": {
|
||||
"version": "4.7.2",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.2.tgz",
|
||||
"integrity": "sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ==",
|
||||
"dependencies": {
|
||||
"buffer": "^5.6.0"
|
||||
},
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-6.2.0.tgz",
|
||||
"integrity": "sha512-ID1cI+7bazPDyL9wYy9GaQ8gEEohWvcUl/Yf0dIdutJxnmInEEyCsb4awy/OiBfall7zBA179Pahi3vCdFze3Q==",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"services/history-v1/node_modules/buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
|
||||
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"base64-js": "^1.3.1",
|
||||
"ieee754": "^1.1.13"
|
||||
"node": ">=16.20.1"
|
||||
}
|
||||
},
|
||||
"services/history-v1/node_modules/check-types": {
|
||||
|
|
@ -38536,21 +38510,63 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"services/history-v1/node_modules/mongodb": {
|
||||
"version": "4.17.1",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.17.1.tgz",
|
||||
"integrity": "sha512-MBuyYiPUPRTqfH2dV0ya4dcr2E5N52ocBuZ8Sgg/M030nGF78v855B3Z27mZJnp8PxjnUquEnAtjOsphgMZOlQ==",
|
||||
"services/history-v1/node_modules/gcp-metadata": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz",
|
||||
"integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"bson": "^4.7.2",
|
||||
"mongodb-connection-string-url": "^2.6.0",
|
||||
"socks": "^2.7.1"
|
||||
"gaxios": "^5.0.0",
|
||||
"json-bigint": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.9.0"
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"services/history-v1/node_modules/mongodb": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.2.0.tgz",
|
||||
"integrity": "sha512-d7OSuGjGWDZ5usZPqfvb36laQ9CPhnWkAGHT61x5P95p/8nMVeH8asloMwW6GcYFeB0Vj4CB/1wOTDG2RA9BFA==",
|
||||
"dependencies": {
|
||||
"@mongodb-js/saslprep": "^1.1.0",
|
||||
"bson": "^6.2.0",
|
||||
"mongodb-connection-string-url": "^2.6.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@aws-sdk/credential-providers": "^3.186.0",
|
||||
"@mongodb-js/saslprep": "^1.1.0"
|
||||
"engines": {
|
||||
"node": ">=16.20.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@aws-sdk/credential-providers": "^3.188.0",
|
||||
"@mongodb-js/zstd": "^1.1.0",
|
||||
"gcp-metadata": "^5.2.0",
|
||||
"kerberos": "^2.0.1",
|
||||
"mongodb-client-encryption": ">=6.0.0 <7",
|
||||
"snappy": "^7.2.2",
|
||||
"socks": "^2.7.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@aws-sdk/credential-providers": {
|
||||
"optional": true
|
||||
},
|
||||
"@mongodb-js/zstd": {
|
||||
"optional": true
|
||||
},
|
||||
"gcp-metadata": {
|
||||
"optional": true
|
||||
},
|
||||
"kerberos": {
|
||||
"optional": true
|
||||
},
|
||||
"mongodb-client-encryption": {
|
||||
"optional": true
|
||||
},
|
||||
"snappy": {
|
||||
"optional": true
|
||||
},
|
||||
"socks": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"services/history-v1/node_modules/test-value": {
|
||||
|
|
@ -65016,7 +65032,7 @@
|
|||
"knex": "^2.4.0",
|
||||
"lodash": "^4.17.19",
|
||||
"mocha": "^10.2.0",
|
||||
"mongodb": "^4.11.0",
|
||||
"mongodb": "^6.2.0",
|
||||
"node-fetch": "^2.6.7",
|
||||
"overleaf-editor-core": "*",
|
||||
"pg": "^8.7.1",
|
||||
|
|
@ -65039,21 +65055,9 @@
|
|||
}
|
||||
},
|
||||
"bson": {
|
||||
"version": "4.7.2",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.2.tgz",
|
||||
"integrity": "sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ==",
|
||||
"requires": {
|
||||
"buffer": "^5.6.0"
|
||||
}
|
||||
},
|
||||
"buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
|
||||
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
|
||||
"requires": {
|
||||
"base64-js": "^1.3.1",
|
||||
"ieee754": "^1.1.13"
|
||||
}
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-6.2.0.tgz",
|
||||
"integrity": "sha512-ID1cI+7bazPDyL9wYy9GaQ8gEEohWvcUl/Yf0dIdutJxnmInEEyCsb4awy/OiBfall7zBA179Pahi3vCdFze3Q=="
|
||||
},
|
||||
"check-types": {
|
||||
"version": "11.1.2",
|
||||
|
|
@ -65091,16 +65095,25 @@
|
|||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "4.17.1",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.17.1.tgz",
|
||||
"integrity": "sha512-MBuyYiPUPRTqfH2dV0ya4dcr2E5N52ocBuZ8Sgg/M030nGF78v855B3Z27mZJnp8PxjnUquEnAtjOsphgMZOlQ==",
|
||||
"gcp-metadata": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz",
|
||||
"integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"gaxios": "^5.0.0",
|
||||
"json-bigint": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.2.0.tgz",
|
||||
"integrity": "sha512-d7OSuGjGWDZ5usZPqfvb36laQ9CPhnWkAGHT61x5P95p/8nMVeH8asloMwW6GcYFeB0Vj4CB/1wOTDG2RA9BFA==",
|
||||
"requires": {
|
||||
"@aws-sdk/credential-providers": "^3.186.0",
|
||||
"@mongodb-js/saslprep": "^1.1.0",
|
||||
"bson": "^4.7.2",
|
||||
"mongodb-connection-string-url": "^2.6.0",
|
||||
"socks": "^2.7.1"
|
||||
"bson": "^6.2.0",
|
||||
"mongodb-connection-string-url": "^2.6.0"
|
||||
}
|
||||
},
|
||||
"test-value": {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@
|
|||
"jsonwebtoken": "^9.0.0",
|
||||
"knex": "^2.4.0",
|
||||
"lodash": "^4.17.19",
|
||||
"mongodb": "^4.11.0",
|
||||
"mongodb": "^6.2.0",
|
||||
"overleaf-editor-core": "*",
|
||||
"pg": "^8.7.1",
|
||||
"swagger-tools": "^0.10.4",
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ async function initialize(projectId) {
|
|||
assert.mongoId(projectId, 'bad projectId')
|
||||
try {
|
||||
await mongodb.blobs.insertOne({
|
||||
_id: ObjectId(projectId),
|
||||
_id: new ObjectId(projectId),
|
||||
blobs: {},
|
||||
})
|
||||
} catch (err) {
|
||||
|
|
@ -48,7 +48,7 @@ async function findBlob(projectId, hash) {
|
|||
|
||||
const bucket = getBucket(hash)
|
||||
const result = await mongodb.blobs.findOne(
|
||||
{ _id: ObjectId(projectId) },
|
||||
{ _id: new ObjectId(projectId) },
|
||||
{ projection: { _id: 0, bucket: `$${bucket}` } }
|
||||
)
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ async function findBlobs(projectId, hashes) {
|
|||
projection[bucket] = 1
|
||||
}
|
||||
const result = await mongodb.blobs.findOne(
|
||||
{ _id: ObjectId(projectId) },
|
||||
{ _id: new ObjectId(projectId) },
|
||||
{ projection }
|
||||
)
|
||||
|
||||
|
|
@ -191,7 +191,7 @@ async function insertBlob(projectId, blob) {
|
|||
const record = blobToRecord(blob)
|
||||
const result = await mongodb.blobs.updateOne(
|
||||
{
|
||||
_id: ObjectId(projectId),
|
||||
_id: new ObjectId(projectId),
|
||||
$expr: {
|
||||
$lt: [{ $size: { $ifNull: [`$${bucket}`, []] } }, MAX_BLOBS_IN_BUCKET],
|
||||
},
|
||||
|
|
@ -224,7 +224,7 @@ async function insertRecordSharded(projectId, hash, record) {
|
|||
*/
|
||||
async function deleteBlobs(projectId) {
|
||||
assert.mongoId(projectId, 'bad projectId')
|
||||
await mongodb.blobs.deleteOne({ _id: ObjectId(projectId) })
|
||||
await mongodb.blobs.deleteOne({ _id: new ObjectId(projectId) })
|
||||
const minShardedId = makeShardedId(projectId, '0')
|
||||
const maxShardedId = makeShardedId(projectId, 'f')
|
||||
await mongodb.shardedBlobs.deleteMany({
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ async function getLatestChunk(projectId) {
|
|||
assert.mongoId(projectId, 'bad projectId')
|
||||
|
||||
const record = await mongodb.chunks.findOne(
|
||||
{ projectId: ObjectId(projectId), state: 'active' },
|
||||
{ projectId: new ObjectId(projectId), state: 'active' },
|
||||
{ sort: { startVersion: -1 } }
|
||||
)
|
||||
if (record == null) {
|
||||
|
|
@ -32,7 +32,7 @@ async function getChunkForVersion(projectId, version) {
|
|||
|
||||
const record = await mongodb.chunks.findOne(
|
||||
{
|
||||
projectId: ObjectId(projectId),
|
||||
projectId: new ObjectId(projectId),
|
||||
state: 'active',
|
||||
startVersion: { $lte: version },
|
||||
endVersion: { $gte: version },
|
||||
|
|
@ -55,7 +55,7 @@ async function getChunkForTimestamp(projectId, timestamp) {
|
|||
|
||||
const record = await mongodb.chunks.findOne(
|
||||
{
|
||||
projectId: ObjectId(projectId),
|
||||
projectId: new ObjectId(projectId),
|
||||
state: 'active',
|
||||
endTimestamp: { $gte: timestamp },
|
||||
},
|
||||
|
|
@ -84,7 +84,7 @@ async function getProjectChunkIds(projectId) {
|
|||
assert.mongoId(projectId, 'bad projectId')
|
||||
|
||||
const cursor = mongodb.chunks.find(
|
||||
{ projectId: ObjectId(projectId), state: 'active' },
|
||||
{ projectId: new ObjectId(projectId), state: 'active' },
|
||||
{ projection: { _id: 1 } }
|
||||
)
|
||||
return await cursor.map(record => record._id).toArray()
|
||||
|
|
@ -100,7 +100,7 @@ async function insertPendingChunk(projectId, chunk) {
|
|||
const chunkId = new ObjectId()
|
||||
await mongodb.chunks.insertOne({
|
||||
_id: chunkId,
|
||||
projectId: ObjectId(projectId),
|
||||
projectId: new ObjectId(projectId),
|
||||
startVersion: chunk.getStartVersion(),
|
||||
endVersion: chunk.getEndVersion(),
|
||||
endTimestamp: chunk.getEndTimestamp(),
|
||||
|
|
@ -122,8 +122,8 @@ async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
|
|||
try {
|
||||
result = await mongodb.chunks.updateOne(
|
||||
{
|
||||
_id: ObjectId(chunkId),
|
||||
projectId: ObjectId(projectId),
|
||||
_id: new ObjectId(chunkId),
|
||||
projectId: new ObjectId(projectId),
|
||||
state: 'pending',
|
||||
},
|
||||
{ $set: { state: 'active', updatedAt: new Date() } },
|
||||
|
|
@ -176,7 +176,7 @@ async function deleteChunk(projectId, chunkId, mongoOpts = {}) {
|
|||
assert.mongoId(chunkId, 'bad chunkId')
|
||||
|
||||
await mongodb.chunks.updateOne(
|
||||
{ _id: ObjectId(chunkId), projectId: ObjectId(projectId) },
|
||||
{ _id: new ObjectId(chunkId), projectId: new ObjectId(projectId) },
|
||||
{ $set: { state: 'deleted', updatedAt: new Date() } },
|
||||
mongoOpts
|
||||
)
|
||||
|
|
@ -189,7 +189,7 @@ async function deleteProjectChunks(projectId) {
|
|||
assert.mongoId(projectId, 'bad projectId')
|
||||
|
||||
await mongodb.chunks.updateMany(
|
||||
{ projectId: ObjectId(projectId), state: 'active' },
|
||||
{ projectId: new ObjectId(projectId), state: 'active' },
|
||||
{ $set: { state: 'deleted', updatedAt: new Date() } }
|
||||
)
|
||||
}
|
||||
|
|
@ -239,7 +239,7 @@ async function getOldChunksBatch(count, minAgeSecs) {
|
|||
*/
|
||||
async function deleteOldChunks(chunkIds) {
|
||||
await mongodb.chunks.deleteMany({
|
||||
_id: { $in: chunkIds.map(ObjectId) },
|
||||
_id: { $in: chunkIds.map(id => new ObjectId(id)) },
|
||||
state: { $in: ['deleted', 'pending'] },
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ const mongoBackend = require('../../../../storage/lib/blob_store/mongo')
|
|||
const mongodb = require('../../../../storage/lib/mongodb')
|
||||
|
||||
describe('BlobStore Mongo backend', function () {
|
||||
const projectId = ObjectId().toString()
|
||||
const projectId = new ObjectId().toString()
|
||||
const hashes = {
|
||||
abcd: [
|
||||
'abcd000000000000000000000000000000000000',
|
||||
|
|
@ -38,7 +38,7 @@ describe('BlobStore Mongo backend', function () {
|
|||
const blob = new Blob(hash, 123, 99)
|
||||
await mongoBackend.insertBlob(projectId, blob)
|
||||
}
|
||||
const record = await mongodb.blobs.findOne(ObjectId(projectId), {
|
||||
const record = await mongodb.blobs.findOne(new ObjectId(projectId), {
|
||||
promoteBuffers: true,
|
||||
})
|
||||
expect(record.blobs).to.deep.equal({
|
||||
|
|
@ -56,7 +56,7 @@ describe('BlobStore Mongo backend', function () {
|
|||
const blob = new Blob(hash, 123, 99)
|
||||
await mongoBackend.insertBlob(projectId, blob)
|
||||
}
|
||||
const record = await mongodb.blobs.findOne(ObjectId(projectId), {
|
||||
const record = await mongodb.blobs.findOne(new ObjectId(projectId), {
|
||||
promoteBuffers: true,
|
||||
})
|
||||
expect(record.blobs).to.deep.equal({
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ describe('chunk store Mongo backend', function () {
|
|||
|
||||
describe('garbage collection', function () {
|
||||
it('deletes pending and deleted chunks', async function () {
|
||||
const projectId = ObjectId().toString()
|
||||
const projectId = new ObjectId().toString()
|
||||
|
||||
// Create a pending chunk
|
||||
const pendingChunk = makeChunk([], 0)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ const cleanup = require('./support/cleanup')
|
|||
|
||||
const CHUNK_STORE_BUCKET = config.get('chunkStore.bucket')
|
||||
const postgresProjectId = 1
|
||||
const mongoProjectId = ObjectId('abcdefabcdefabcdefabcdef')
|
||||
const mongoProjectId = new ObjectId('abcdefabcdefabcdefabcdef')
|
||||
|
||||
describe('tasks', function () {
|
||||
beforeEach(cleanup.everything)
|
||||
|
|
@ -42,7 +42,7 @@ describe('tasks', function () {
|
|||
deleted_at: deletedAt,
|
||||
})
|
||||
mongoChunks.push({
|
||||
_id: ObjectId(i.toString().padStart(24, '0')),
|
||||
_id: new ObjectId(i.toString().padStart(24, '0')),
|
||||
projectId: mongoProjectId,
|
||||
startVersion,
|
||||
endVersion,
|
||||
|
|
@ -63,7 +63,7 @@ describe('tasks', function () {
|
|||
deleted_at: deletedAt,
|
||||
})
|
||||
mongoChunks.push({
|
||||
_id: ObjectId(i.toString().padStart(24, '0')),
|
||||
_id: new ObjectId(i.toString().padStart(24, '0')),
|
||||
projectId: mongoProjectId,
|
||||
startVersion,
|
||||
endVersion,
|
||||
|
|
|
|||
Loading…
Reference in a new issue