github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-10-17 21:05:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #20297 from overleaf/mj-stream-bib-log

Browse source 
[web] Stream .blg files before parsing

GitOrigin-RevId: 2e7a16a19e62fc32bbcb7fe35bed745b938e6963
This commit is contained in:
Mathias Jakobsen 2024-09-17 11:47:46 +01:00 committed by Copybot
parent 3b839dc885
commit a807a810e6
1 changed files with 46 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
services/web/frontend/js/features/pdf-preview/util/output-files.js
View file

@ -9,6 +9,7 @@ import '@/utils/readable-stream-async-iterator-polyfill'
const TRANSIENT_WARNING_REGEX = /^(Reference|Citation).+undefined on input line/
const MAX_LOG_SIZE = 1024 * 1024 // 1MB
const MAX_BIB_LOG_SIZE_PER_FILE = MAX_LOG_SIZE
export function handleOutputFiles(outputFiles, projectId, data) {
const outputFile = outputFiles.get('output.pdf')
@ -77,31 +78,11 @@ export const handleLogFiles = async (outputFiles, data, signal) => {
const logFile = outputFiles.get('output.log')
if (logFile) {
try {
const logFileAbortController = new AbortController()
// abort fetching the log file if the main signal is aborted
signal.addEventListener('abort', () => {
logFileAbortController.abort()
})
const response = await fetch(buildURL(logFile, data.pdfDownloadDomain), {
signal: logFileAbortController.signal,
})
result.log = ''
const reader = response.body.pipeThrough(new TextDecoderStream())
for await (const chunk of reader) {
result.log += chunk
if (result.log.length > MAX_LOG_SIZE) {
logFileAbortController.abort()
}
}
} catch (e) {
debugConsole.warn(e) // ignore failure to fetch the log file, but log a warning
}
result.log = await fetchFileWithSizeLimit(
buildURL(logFile, data.pdfDownloadDomain),
signal,
MAX_LOG_SIZE
)
try {
let { errors, warnings, typesetting } = HumanReadableLogs.parse(
result.log,
@ -129,13 +110,11 @@ export const handleLogFiles = async (outputFiles, data, signal) => {
}
}
for (const blgFile of blgFiles) {
try {
const response = await fetch(buildURL(blgFile, data.pdfDownloadDomain), {
const log = await fetchFileWithSizeLimit(
buildURL(blgFile, data.pdfDownloadDomain),
signal,
})
const log = await response.text()
MAX_BIB_LOG_SIZE_PER_FILE
)
try {
const { errors, warnings } = new BibLogParser(log, {
maxErrors: 100,
@ -144,9 +123,6 @@ export const handleLogFiles = async (outputFiles, data, signal) => {
} catch (e) {
// BibLog parsing errors are ignored
}
} catch (e) {
debugConsole.warn(e) // ignore failure to fetch/parse the log file, but log a warning
}
}
result.logEntries.all = [
@ -260,3 +236,33 @@ function normalizeFilePath(path, rootDocDirname) {
function isTransientWarning(warning) {
return TRANSIENT_WARNING_REGEX.test(warning.message)
}
async function fetchFileWithSizeLimit(url, signal, maxSize) {
let result = ''
try {
const abortController = new AbortController()
// abort fetching the log file if the main signal is aborted
signal.addEventListener('abort', () => {
abortController.abort()
})
const response = await fetch(url, {
signal: abortController.signal,
})
if (!response.ok) {
throw new Error('Failed to fetch log file')
}
const reader = response.body.pipeThrough(new TextDecoderStream())
for await (const chunk of reader) {
result += chunk
if (result.length > maxSize) {
abortController.abort()
}
}
} catch (e) {
debugConsole.warn(e) // ignore failure to fetch the log file, but log a warning
}
return result
}