github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #5168 from overleaf/em-request-pipe-linked-files

Browse source 
Fix linked files when they refer to empty files

GitOrigin-RevId: 13c751e2679849516e874afdd422cf9e8d16c8d1
This commit is contained in:
Eric Mc Sween 2021-09-22 08:32:16 -04:00 committed by Copybot
parent 59d83c4b88
commit a163efd0a1
2 changed files with 284 additions and 366 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

276
services/web/app/src/Features/LinkedFiles/ProjectOutputFileAgent.js
View file

@ -1,21 +1,4 @@
/* eslint-disable
camelcase,
node/handle-callback-err,
max-len,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ProjectOutputFileAgent
const AuthorizationManager = require('../Authorization/AuthorizationManager') const AuthorizationManager = require('../Authorization/AuthorizationManager')
const ProjectGetter = require('../Project/ProjectGetter')
const Settings = require('@overleaf/settings')
const CompileManager = require('../Compile/CompileManager') const CompileManager = require('../Compile/CompileManager')
const ClsiManager = require('../Compile/ClsiManager') const ClsiManager = require('../Compile/ClsiManager')
const ProjectFileAgent = require('./ProjectFileAgent') const ProjectFileAgent = require('./ProjectFileAgent')
@ -24,79 +7,60 @@ const {
CompileFailedError, CompileFailedError,
BadDataError, BadDataError,
AccessDeniedError, AccessDeniedError,
BadEntityTypeError,
OutputFileFetchFailedError, OutputFileFetchFailedError,
} = require('./LinkedFilesErrors') } = require('./LinkedFilesErrors')
const LinkedFilesHandler = require('./LinkedFilesHandler') const LinkedFilesHandler = require('./LinkedFilesHandler')
const logger = require('logger-sharelatex')
module.exports = ProjectOutputFileAgent = { function _prepare(projectId, linkedFileData, userId, callback) {
_prepare(project_id, linkedFileData, user_id, callback) { _checkAuth(projectId, linkedFileData, userId, (err, allowed) => {
if (callback == null) { if (err) {
callback = function (err, linkedFileData) {}
}
return this._checkAuth(
project_id,
linkedFileData,
user_id,
(err, allowed) => {
if (err != null) {
return callback(err) return callback(err)
} }
if (!allowed) { if (!allowed) {
return callback(new AccessDeniedError()) return callback(new AccessDeniedError())
} }
if (!this._validate(linkedFileData)) { if (!_validate(linkedFileData)) {
return callback(new BadDataError()) return callback(new BadDataError())
} }
return callback(null, linkedFileData) callback(null, linkedFileData)
} })
) }
},
createLinkedFile( function createLinkedFile(
project_id, projectId,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
callback callback
) { ) {
if (!this._canCreate(linkedFileData)) { if (!ProjectFileAgent._canCreate(linkedFileData)) {
return callback(new AccessDeniedError()) return callback(new AccessDeniedError())
} }
linkedFileData = this._sanitizeData(linkedFileData) linkedFileData = _sanitizeData(linkedFileData)
return this._prepare( _prepare(projectId, linkedFileData, userId, (err, linkedFileData) => {
project_id, if (err) {
linkedFileData,
user_id,
(err, linkedFileData) => {
if (err != null) {
return callback(err) return callback(err)
} }
return this._getFileStream( _getFileStream(linkedFileData, userId, (err, readStream) => {
linkedFileData, if (err) {
user_id,
(err, readStream) => {
if (err != null) {
return callback(err) return callback(err)
} }
readStream.on('error', callback) readStream.on('error', callback)
return readStream.on('response', response => { readStream.on('response', response => {
if (response.statusCode >= 200 && response.statusCode < 300) { if (response.statusCode >= 200 && response.statusCode < 300) {
readStream.resume() LinkedFilesHandler.importFromStream(
return LinkedFilesHandler.importFromStream( projectId,
project_id,
readStream, readStream,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
function (err, file) { (err, file) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
return callback(null, file._id) callback(null, file._id)
} }
) // Created ) // Created
} else { } else {
@ -104,55 +68,48 @@ module.exports = ProjectOutputFileAgent = {
`Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}` `Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}`
) )
err.statusCode = response.statusCode err.statusCode = response.statusCode
return callback(err) callback(err)
} }
}) })
} })
) })
} }
)
},
refreshLinkedFile( function refreshLinkedFile(
project_id, projectId,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
callback callback
) { ) {
return this._prepare( _prepare(projectId, linkedFileData, userId, (err, linkedFileData) => {
project_id, if (err) {
linkedFileData,
user_id,
(err, linkedFileData) => {
if (err != null) {
return callback(err) return callback(err)
} }
return this._compileAndGetFileStream( _compileAndGetFileStream(
linkedFileData, linkedFileData,
user_id, userId,
(err, readStream, new_build_id) => { (err, readStream, newBuildId) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
readStream.on('error', callback) readStream.on('error', callback)
return readStream.on('response', response => { readStream.on('response', response => {
if (response.statusCode >= 200 && response.statusCode < 300) { if (response.statusCode >= 200 && response.statusCode < 300) {
readStream.resume() linkedFileData.build_id = newBuildId
linkedFileData.build_id = new_build_id LinkedFilesHandler.importFromStream(
return LinkedFilesHandler.importFromStream( projectId,
project_id,
readStream, readStream,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
function (err, file) { (err, file) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
return callback(null, file._id) callback(null, file._id)
} }
) // Created ) // Created
} else { } else {
@ -160,112 +117,96 @@ module.exports = ProjectOutputFileAgent = {
`Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}` `Output file fetch failed: ${linkedFileData.build_id}, ${linkedFileData.source_output_file_path}`
) )
err.statusCode = response.statusCode err.statusCode = response.statusCode
return callback(err) callback(err)
} }
}) })
} }
) )
} })
) }
},
_sanitizeData(data) { function _sanitizeData(data) {
return { return {
provider: data.provider, provider: data.provider,
source_project_id: data.source_project_id, source_project_id: data.source_project_id,
source_output_file_path: data.source_output_file_path, source_output_file_path: data.source_output_file_path,
build_id: data.build_id, build_id: data.build_id,
} }
}, }
_canCreate: ProjectFileAgent._canCreate, function _validate(data) {
_getSourceProject: LinkedFilesHandler.getSourceProject,
_validate(data) {
if (data.v1_source_doc_id != null) {
return ( return (
data.v1_source_doc_id != null && data.source_output_file_path != null (data.v1_source_doc_id != null && data.source_output_file_path != null) ||
) (data.source_project_id != null &&
} else {
return (
data.source_project_id != null &&
data.source_output_file_path != null && data.source_output_file_path != null &&
data.build_id != null data.build_id != null)
) )
} }
},
_checkAuth(project_id, data, current_user_id, callback) { function _checkAuth(projectId, data, currentUserId, callback) {
if (callback == null) {
callback = function (err, allowed) {}
}
callback = _.once(callback) callback = _.once(callback)
if (!this._validate(data)) { if (!_validate(data)) {
return callback(new BadDataError()) return callback(new BadDataError())
} }
return this._getSourceProject(data, function (err, project) { LinkedFilesHandler.getSourceProject(data, (err, project) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
return AuthorizationManager.canUserReadProject( AuthorizationManager.canUserReadProject(
current_user_id, currentUserId,
project._id, project._id,
null, null,
function (err, canRead) { (err, canRead) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
return callback(null, canRead) callback(null, canRead)
} }
) )
}) })
}, }
_getFileStream(linkedFileData, user_id, callback) { function _getFileStream(linkedFileData, userId, callback) {
if (callback == null) {
callback = function (err, fileStream) {}
}
callback = _.once(callback) callback = _.once(callback)
const { source_output_file_path, build_id } = linkedFileData const {
return this._getSourceProject(linkedFileData, function (err, project) { source_output_file_path: sourceOutputFilePath,
if (err != null) { build_id: buildId,
} = linkedFileData
LinkedFilesHandler.getSourceProject(linkedFileData, (err, project) => {
if (err) {
return callback(err) return callback(err)
} }
const source_project_id = project._id const sourceProjectId = project._id
return ClsiManager.getOutputFileStream( ClsiManager.getOutputFileStream(
source_project_id, sourceProjectId,
user_id, userId,
build_id, buildId,
source_output_file_path, sourceOutputFilePath,
function (err, readStream) { (err, readStream) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
readStream.pause() readStream.pause()
return callback(null, readStream) callback(null, readStream)
} }
) )
}) })
}, }
_compileAndGetFileStream(linkedFileData, user_id, callback) { function _compileAndGetFileStream(linkedFileData, userId, callback) {
if (callback == null) {
callback = function (err, stream, build_id) {}
}
callback = _.once(callback) callback = _.once(callback)
const { source_output_file_path } = linkedFileData const { source_output_file_path: sourceOutputFilePath } = linkedFileData
return this._getSourceProject(linkedFileData, function (err, project) { LinkedFilesHandler.getSourceProject(linkedFileData, (err, project) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
const source_project_id = project._id const sourceProjectId = project._id
return CompileManager.compile( CompileManager.compile(
source_project_id, sourceProjectId,
user_id, userId,
{}, {},
function (err, status, outputFiles) { (err, status, outputFiles) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
if (status !== 'success') { if (status !== 'success') {
@ -273,27 +214,28 @@ module.exports = ProjectOutputFileAgent = {
} }
const outputFile = _.find( const outputFile = _.find(
outputFiles, outputFiles,
o => o.path === source_output_file_path o => o.path === sourceOutputFilePath
) )
if (outputFile == null) { if (outputFile == null) {
return callback(new OutputFileFetchFailedError()) return callback(new OutputFileFetchFailedError())
} }
const build_id = outputFile.build const buildId = outputFile.build
return ClsiManager.getOutputFileStream( ClsiManager.getOutputFileStream(
source_project_id, sourceProjectId,
user_id, userId,
build_id, buildId,
source_output_file_path, sourceOutputFilePath,
function (err, readStream) { (err, readStream) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
readStream.pause() readStream.pause()
return callback(null, readStream, build_id) callback(null, readStream, buildId)
} }
) )
} }
) )
}) })
},
} }
module.exports = { createLinkedFile, refreshLinkedFile }

94
services/web/app/src/Features/LinkedFiles/UrlAgent.js
View file

@ -1,18 +1,3 @@
/* eslint-disable
camelcase,
node/handle-callback-err,
max-len,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let UrlAgent
const request = require('request') const request = require('request')
const _ = require('underscore') const _ = require('underscore')
const urlValidator = require('valid-url') const urlValidator = require('valid-url')
@ -20,40 +5,34 @@ const { InvalidUrlError, UrlFetchFailedError } = require('./LinkedFilesErrors')
const LinkedFilesHandler = require('./LinkedFilesHandler') const LinkedFilesHandler = require('./LinkedFilesHandler')
const UrlHelper = require('../Helpers/UrlHelper') const UrlHelper = require('../Helpers/UrlHelper')
module.exports = UrlAgent = { function createLinkedFile(
createLinkedFile( projectId,
project_id,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
callback callback
) { ) {
linkedFileData = this._sanitizeData(linkedFileData) linkedFileData = _sanitizeData(linkedFileData)
return this._getUrlStream( _getUrlStream(projectId, linkedFileData, userId, (err, readStream) => {
project_id, if (err) {
linkedFileData,
user_id,
function (err, readStream) {
if (err != null) {
return callback(err) return callback(err)
} }
readStream.on('error', callback) readStream.on('error', callback)
return readStream.on('response', function (response) { readStream.on('response', response => {
if (response.statusCode >= 200 && response.statusCode < 300) { if (response.statusCode >= 200 && response.statusCode < 300) {
readStream.resume() LinkedFilesHandler.importFromStream(
return LinkedFilesHandler.importFromStream( projectId,
project_id,
readStream, readStream,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
function (err, file) { (err, file) => {
if (err != null) { if (err) {
return callback(err) return callback(err)
} }
return callback(null, file._id) callback(null, file._id)
} }
) // Created ) // Created
} else { } else {
@ -61,42 +40,38 @@ module.exports = UrlAgent = {
`url fetch failed: ${linkedFileData.url}` `url fetch failed: ${linkedFileData.url}`
) )
error.statusCode = response.statusCode error.statusCode = response.statusCode
return callback(error) callback(error)
} }
}) })
} })
) }
},
refreshLinkedFile( function refreshLinkedFile(
project_id, projectId,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
callback callback
) { ) {
return this.createLinkedFile( createLinkedFile(
project_id, projectId,
linkedFileData, linkedFileData,
name, name,
parent_folder_id, parentFolderId,
user_id, userId,
callback callback
) )
}, }
_sanitizeData(data) { function _sanitizeData(data) {
return { return {
provider: data.provider, provider: data.provider,
url: UrlHelper.prependHttpIfNeeded(data.url), url: UrlHelper.prependHttpIfNeeded(data.url),
} }
}, }
_getUrlStream(project_id, data, current_user_id, callback) { function _getUrlStream(projectId, data, currentUserId, callback) {
if (callback == null) {
callback = function (error, fsPath) {}
}
callback = _.once(callback) callback = _.once(callback)
let { url } = data let { url } = data
if (!urlValidator.isWebUri(url)) { if (!urlValidator.isWebUri(url)) {
@ -105,6 +80,7 @@ module.exports = UrlAgent = {
url = UrlHelper.wrapUrlWithProxy(url) url = UrlHelper.wrapUrlWithProxy(url)
const readStream = request.get(url) const readStream = request.get(url)
readStream.pause() readStream.pause()
return callback(null, readStream) callback(null, readStream)
},
} }
module.exports = { createLinkedFile, refreshLinkedFile }