github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-19 16:15:25 +00:00
Code Issues Projects Releases Packages Wiki Activity

Improve chunk loading in backupVerifier

Browse source 
Brings the process closer to history_store.

We can't use the backup history_store because the keys are generated
differently for chunks than the standard history_store way of doing it.

GitOrigin-RevId: 07adfc0531f6ec0f38bb70ea0fe8ae0d41f508cc
This commit is contained in:
Andrew Rumble 2025-03-05 17:02:55 +00:00 • committed by Copybot
parent d04bfbcdea
commit 36056e75d7
1 changed files with 20 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
services/history-v1/storage/lib/backupVerifier.mjs
View file

@ -12,10 +12,10 @@ import { BlobStore, GLOBAL_BLOBS, makeProjectKey } from './blob_store/index.js'
import blobHash from './blob_hash.js'
import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
import logger from '@overleaf/logger'
import { text } from 'node:stream/consumers'
import { createGunzip } from 'node:zlib'
import path from 'node:path'
import projectKey from './project_key.js'
import streams from './streams.js'
import objectPersistor from '@overleaf/object-persistor'
const RPO = parseInt(config.get('backupRPOInMS'), 10)
@ -123,12 +123,20 @@ async function loadChunk(historyId, startVersion, backupPersistorForProject) {
projectKey.format(historyId),
projectKey.pad(startVersion)
)
const backupChunkStream = await backupPersistorForProject.getObjectStream(
chunksBucket,
key
)
const raw = await text(backupChunkStream.pipe(createGunzip()))
return JSON.parse(raw)
try {
const buf = await streams.gunzipStreamToBuffer(
await backupPersistorForProject.getObjectStream(chunksBucket, key)
)
return JSON.parse(buf.toString('utf-8'))
} catch (err) {
if (err instanceof objectPersistor.Errors.NotFoundError) {
throw new Chunk.NotPersistedError(historyId)
}
if (err instanceof Error) {
throw OError.tag(err, 'Failed to load chunk', { historyId, startVersion })
}
throw err
}
}
/**
@ -165,7 +173,10 @@ export async function verifyProject(historyId, endTimestamp) {
)
} catch (err) {
if (err instanceof Chunk.NotPersistedError) {
throw new BackupRPOViolationError('backup RPO violation', chunk)
throw new BackupRPOViolationError(
'BackupRPOviolation: chunk not backed up',
chunk
)
}
throw err
}