mirror of
https://github.com/overleaf/overleaf.git
synced 2024-12-04 23:26:02 -05:00
a807a810e6
[web] Stream .blg files before parsing GitOrigin-RevId: 2e7a16a19e62fc32bbcb7fe35bed745b938e6963
268 lines
7.3 KiB
JavaScript
268 lines
7.3 KiB
JavaScript
import HumanReadableLogs from '../../../ide/human-readable-logs/HumanReadableLogs'
|
|
import BibLogParser from '../../../ide/log-parser/bib-log-parser'
|
|
import { enablePdfCaching } from './pdf-caching-flags'
|
|
import { debugConsole } from '@/utils/debugging'
|
|
import { dirname, findEntityByPath } from '@/features/file-tree/util/path'
|
|
import '@/utils/readable-stream-async-iterator-polyfill'
|
|
|
|
// Warnings that may disappear after a second LaTeX pass
|
|
const TRANSIENT_WARNING_REGEX = /^(Reference|Citation).+undefined on input line/
|
|
|
|
const MAX_LOG_SIZE = 1024 * 1024 // 1MB
|
|
const MAX_BIB_LOG_SIZE_PER_FILE = MAX_LOG_SIZE
|
|
|
|
export function handleOutputFiles(outputFiles, projectId, data) {
|
|
const outputFile = outputFiles.get('output.pdf')
|
|
if (!outputFile) return null
|
|
|
|
// build the URL for viewing the PDF in the preview UI
|
|
const params = new URLSearchParams({
|
|
compileGroup: data.compileGroup,
|
|
})
|
|
|
|
if (data.clsiServerId) {
|
|
params.set('clsiserverid', data.clsiServerId)
|
|
}
|
|
|
|
if (enablePdfCaching) {
|
|
// Tag traffic that uses the pdf caching logic.
|
|
params.set('enable_pdf_caching', 'true')
|
|
}
|
|
|
|
outputFile.pdfUrl = `${buildURL(
|
|
outputFile,
|
|
data.pdfDownloadDomain
|
|
)}?${params}`
|
|
|
|
// build the URL for downloading the PDF
|
|
params.set('popupDownload', 'true') // save PDF download as file
|
|
|
|
outputFile.pdfDownloadUrl = `/download/project/${projectId}/build/${outputFile.build}/output/output.pdf?${params}`
|
|
|
|
return outputFile
|
|
}
|
|
|
|
let nextEntryId = 1
|
|
|
|
function generateEntryKey() {
|
|
return 'compile-log-entry-' + nextEntryId++
|
|
}
|
|
|
|
export const handleLogFiles = async (outputFiles, data, signal) => {
|
|
const result = {
|
|
log: null,
|
|
logEntries: {
|
|
errors: [],
|
|
warnings: [],
|
|
typesetting: [],
|
|
},
|
|
}
|
|
|
|
function accumulateResults(newEntries, type) {
|
|
for (const key in result.logEntries) {
|
|
if (newEntries[key]) {
|
|
for (const entry of newEntries[key]) {
|
|
if (type) {
|
|
entry.type = newEntries.type
|
|
}
|
|
if (entry.file) {
|
|
entry.file = normalizeFilePath(entry.file)
|
|
}
|
|
entry.key = generateEntryKey()
|
|
}
|
|
result.logEntries[key].push(...newEntries[key])
|
|
}
|
|
}
|
|
}
|
|
|
|
const logFile = outputFiles.get('output.log')
|
|
|
|
if (logFile) {
|
|
result.log = await fetchFileWithSizeLimit(
|
|
buildURL(logFile, data.pdfDownloadDomain),
|
|
signal,
|
|
MAX_LOG_SIZE
|
|
)
|
|
try {
|
|
let { errors, warnings, typesetting } = HumanReadableLogs.parse(
|
|
result.log,
|
|
{
|
|
ignoreDuplicates: true,
|
|
}
|
|
)
|
|
|
|
if (data.status === 'stopped-on-first-error') {
|
|
// Hide warnings that could disappear after a second pass
|
|
warnings = warnings.filter(warning => !isTransientWarning(warning))
|
|
}
|
|
|
|
accumulateResults({ errors, warnings, typesetting })
|
|
} catch (e) {
|
|
debugConsole.warn(e) // ignore failure to parse the log file, but log a warning
|
|
}
|
|
}
|
|
|
|
const blgFiles = []
|
|
|
|
for (const [filename, file] of outputFiles) {
|
|
if (filename.endsWith('.blg')) {
|
|
blgFiles.push(file)
|
|
}
|
|
}
|
|
for (const blgFile of blgFiles) {
|
|
const log = await fetchFileWithSizeLimit(
|
|
buildURL(blgFile, data.pdfDownloadDomain),
|
|
signal,
|
|
MAX_BIB_LOG_SIZE_PER_FILE
|
|
)
|
|
try {
|
|
const { errors, warnings } = new BibLogParser(log, {
|
|
maxErrors: 100,
|
|
}).parse()
|
|
accumulateResults({ errors, warnings }, 'BibTeX:')
|
|
} catch (e) {
|
|
// BibLog parsing errors are ignored
|
|
}
|
|
}
|
|
|
|
result.logEntries.all = [
|
|
...result.logEntries.errors,
|
|
...result.logEntries.warnings,
|
|
...result.logEntries.typesetting,
|
|
]
|
|
|
|
return result
|
|
}
|
|
|
|
export function buildLogEntryAnnotations(entries, fileTreeData, rootDocId) {
|
|
const rootDocDirname = dirname(fileTreeData, rootDocId)
|
|
|
|
const logEntryAnnotations = {}
|
|
const seenLine = {}
|
|
|
|
for (const entry of entries) {
|
|
if (entry.file) {
|
|
entry.file = normalizeFilePath(entry.file, rootDocDirname)
|
|
|
|
const entity = findEntityByPath(fileTreeData, entry.file)?.entity
|
|
|
|
if (entity) {
|
|
if (!(entity._id in logEntryAnnotations)) {
|
|
logEntryAnnotations[entity._id] = []
|
|
}
|
|
|
|
const annotation = {
|
|
id: entry.key,
|
|
entryIndex: logEntryAnnotations[entity._id].length, // used for maintaining the order of items on the same line
|
|
row: entry.line - 1,
|
|
type: entry.level === 'error' ? 'error' : 'warning',
|
|
text: entry.message,
|
|
source: 'compile', // NOTE: this is used in Ace for filtering the annotations
|
|
ruleId: entry.ruleId,
|
|
command: entry.command,
|
|
}
|
|
|
|
// set firstOnLine for the first non-typesetting annotation on a line
|
|
if (entry.level !== 'typesetting') {
|
|
if (!seenLine[entry.line]) {
|
|
annotation.firstOnLine = true
|
|
seenLine[entry.line] = true
|
|
}
|
|
}
|
|
|
|
logEntryAnnotations[entity._id].push(annotation)
|
|
}
|
|
}
|
|
}
|
|
|
|
return logEntryAnnotations
|
|
}
|
|
|
|
export const buildRuleCounts = (entries = []) => {
|
|
const counts = {}
|
|
for (const entry of entries) {
|
|
const key = `${entry.level}_${entry.ruleId}`
|
|
counts[key] = counts[key] ? counts[key] + 1 : 1
|
|
}
|
|
return counts
|
|
}
|
|
|
|
export const buildRuleDeltas = (ruleCounts, previousRuleCounts) => {
|
|
const counts = {}
|
|
|
|
// keys that are defined in the current log entries
|
|
for (const [key, value] of Object.entries(ruleCounts)) {
|
|
const previousValue = previousRuleCounts[key] ?? 0
|
|
counts[`delta_${key}`] = value - previousValue
|
|
}
|
|
|
|
// keys that are no longer defined in the current log entries
|
|
for (const [key, value] of Object.entries(previousRuleCounts)) {
|
|
if (!(key in ruleCounts)) {
|
|
counts[key] = 0
|
|
counts[`delta_${key}`] = -value
|
|
}
|
|
}
|
|
|
|
return counts
|
|
}
|
|
|
|
function buildURL(file, pdfDownloadDomain) {
|
|
if (file.build && pdfDownloadDomain) {
|
|
// Downloads from the compiles domain must include a build id.
|
|
// The build id is used implicitly for access control.
|
|
return `${pdfDownloadDomain}${file.url}`
|
|
}
|
|
// Go through web instead, which uses mongo for checking project access.
|
|
return `${window.origin}${file.url}`
|
|
}
|
|
|
|
function normalizeFilePath(path, rootDocDirname) {
|
|
path = path.replace(/\/\//g, '/')
|
|
path = path.replace(
|
|
/^.*\/compiles\/[0-9a-f]{24}(-[0-9a-f]{24})?\/(\.\/)?/,
|
|
''
|
|
)
|
|
|
|
path = path.replace(/^\/compile\//, '')
|
|
|
|
if (rootDocDirname) {
|
|
path = path.replace(/^\.\//, rootDocDirname + '/')
|
|
}
|
|
|
|
return path
|
|
}
|
|
|
|
function isTransientWarning(warning) {
|
|
return TRANSIENT_WARNING_REGEX.test(warning.message)
|
|
}
|
|
|
|
async function fetchFileWithSizeLimit(url, signal, maxSize) {
|
|
let result = ''
|
|
try {
|
|
const abortController = new AbortController()
|
|
// abort fetching the log file if the main signal is aborted
|
|
signal.addEventListener('abort', () => {
|
|
abortController.abort()
|
|
})
|
|
|
|
const response = await fetch(url, {
|
|
signal: abortController.signal,
|
|
})
|
|
|
|
if (!response.ok) {
|
|
throw new Error('Failed to fetch log file')
|
|
}
|
|
|
|
const reader = response.body.pipeThrough(new TextDecoderStream())
|
|
for await (const chunk of reader) {
|
|
result += chunk
|
|
if (result.length > maxSize) {
|
|
abortController.abort()
|
|
}
|
|
}
|
|
} catch (e) {
|
|
debugConsole.warn(e) // ignore failure to fetch the log file, but log a warning
|
|
}
|
|
return result
|
|
}
|