github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-30 06:55:22 -05:00
Code Issues Projects Releases Packages Wiki Activity

First pass at URL based linked files

Browse source
This commit is contained in:
James Allen 2018-02-14 15:12:46 +00:00
parent 3bbd49c7eb
commit 85f25b810c
15 changed files with 217 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
services/web/app/coffee/Features/Editor/EditorController.coffee
View file

@ -23,11 +23,11 @@ module.exports = EditorController =
EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
callback(err, doc)
addFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (error, file)->)->
addFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (error, file)->)->
fileName = fileName.trim()
logger.log {project_id, folder_id, fileName, fsPath}, "sending new file to project"
logger.log {project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id}, "sending new file to project"
Metrics.inc "editor.add-file"
ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, user_id, (err, fileRef, folder_id)=>
ProjectEntityUpdateHandler.addFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, fileRef, folder_id)=>
if err?
logger.err err:err, project_id:project_id, folder_id:folder_id, fileName:fileName, "error adding file without lock"
return callback(err)
@ -40,8 +40,8 @@ module.exports = EditorController =
EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
callback err, doc
upsertFile: (project_id, folder_id, fileName, fsPath, source, user_id, callback = (err, file) ->) ->
ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, user_id, (err, file, didAddFile) ->
upsertFile: (project_id, folder_id, fileName, fsPath, linkedFileData, source, user_id, callback = (err, file) ->) ->
ProjectEntityUpdateHandler.upsertFile project_id, folder_id, fileName, fsPath, linkedFileData, user_id, (err, file, didAddFile) ->
return callback(err) if err?
if didAddFile
EditorRealTimeController.emitToRoom project_id, 'reciveNewFile', folder_id, file, source
@ -56,8 +56,8 @@ module.exports = EditorController =
EditorRealTimeController.emitToRoom project_id, 'reciveNewDoc', lastFolder._id, doc, source
callback()
upsertFileWithPath: (project_id, elementPath, fsPath, source, user_id, callback) ->
ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
upsertFileWithPath: (project_id, elementPath, fsPath, linkedFileData, source, user_id, callback) ->
ProjectEntityUpdateHandler.upsertFileWithPath project_id, elementPath, fsPath, linkedFileData, user_id, (err, file, didAddFile, newFolders, lastFolder) ->
return callback(err) if err?
EditorController._notifyProjectUsersOfNewFolders project_id, newFolders, (err) ->
return callback(err) if err?

25
services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee Normal file
View file

@ -0,0 +1,25 @@
AuthenticationController = require '../Authentication/AuthenticationController'
EditorController = require '../Editor/EditorController'
module.exports = LinkedFilesController = {
Agents: {
url: require('./UrlAgent')
}
createLinkedFile: (req, res, next) ->
{project_id} = req.params
{name, provider, data, parent_folder_id} = req.body
user_id = AuthenticationController.getLoggedInUserId(req)
if !LinkedFilesController.Agents.hasOwnProperty(provider)
return res.send(400)
Agent = LinkedFilesController.Agents[provider]
linkedFileData = Agent.sanitizeData(data)
linkedFileData.provider = provider
Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
return next(error) if error?
EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error) ->
return next(error) if error?
res.send(204) # created
}

11
services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee Normal file
View file

@ -0,0 +1,11 @@
AuthorizationMiddlewear = require('../Authorization/AuthorizationMiddlewear')
AuthenticationController = require('../Authentication/AuthenticationController')
LinkedFilesController = require "./LinkedFilesController"
module.exports =
apply: (webRouter) ->
webRouter.post '/project/:project_id/linked_file',
AuthenticationController.requireLogin(),
AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
LinkedFilesController.createLinkedFile

15
services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee Normal file
View file

@ -0,0 +1,15 @@
request = require 'request'
FileWriter = require('../../infrastructure/FileWriter')
module.exports = UrlAgent = {
sanitizeData: (data) ->
return {
url: data.url
}
writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
# TODO: proxy through external API
url = data.url
readStream = request.get(url)
FileWriter.writeStreamToDisk project_id, readStream, callback
}

2
services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee
View file

@ -79,7 +79,7 @@ module.exports = ProjectCreationHandler =
callback(error)
(callback) ->
universePath = Path.resolve(__dirname + "/../../../templates/project_files/universe.jpg")
ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, owner_id, callback
ProjectEntityUpdateHandler.addFile project._id, project.rootFolder[0]._id, "universe.jpg", universePath, null, owner_id, callback
], (error) ->
callback(error, project)

3
services/web/app/coffee/Features/Project/ProjectEntityMongoUpdateHandler.coffee
View file

@ -48,7 +48,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
self._confirmFolder project, folder_id, (folder_id)->
self._putElement project, folder_id, fileRef, "file", callback
replaceFile: wrapWithLock (project_id, file_id, callback) ->
replaceFile: wrapWithLock (project_id, file_id, linkedFileData, callback) ->
ProjectGetter.getProjectWithoutLock project_id, {rootFolder: true, name:true}, (err, project) ->
return callback(err) if err?
ProjectLocator.findElement {project:project, element_id: file_id, type: 'file'}, (err, fileRef, path)=>
@ -63,6 +63,7 @@ module.exports = ProjectEntityMongoUpdateHandler = self =
inc['version'] = 1
set = {}
set["#{path.mongo}.created"] = new Date()
set["#{path.mongo}.linkedFileData"] = linkedFileData
update =
"$inc": inc
"$set": set

25
services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee
View file

@ -129,8 +129,8 @@ module.exports = ProjectEntityUpdateHandler = self =
return callback(error) if error?
callback null, doc, folder_id
addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id) ->)->
self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
addFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id) ->)->
self.addFileWithoutUpdatingHistory.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (error, fileRef, folder_id, path, fileStoreUrl) ->
return callback(error) if error?
newFiles = [
file: fileRef
@ -141,10 +141,10 @@ module.exports = ProjectEntityUpdateHandler = self =
return callback(error) if error?
callback null, fileRef, folder_id
replaceFile: wrapWithLock (project_id, file_id, fsPath, userId, callback)->
replaceFile: wrapWithLock (project_id, file_id, fsPath, linkedFileData, userId, callback)->
FileStoreHandler.uploadFileFromDisk project_id, file_id, fsPath, (err, fileStoreUrl)->
return callback(err) if err?
ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, (err, fileRef, project, path) ->
ProjectEntityMongoUpdateHandler.replaceFile project_id, file_id, linkedFileData, (err, fileRef, project, path) ->
return callback(err) if err?
newFiles = [
file: fileRef
@ -180,7 +180,7 @@ module.exports = ProjectEntityUpdateHandler = self =
return callback(err) if err?
callback(null, doc, folder_id, result?.path?.fileSystem)
addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
addFileWithoutUpdatingHistory: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (error, fileRef, folder_id, path, fileStoreUrl) ->)->
# This method should never be called directly, except when importing a project
# from Overleaf. It skips sending updates to the project history, which will break
# the history unless you are making sure it is updated in some other way.
@ -188,7 +188,10 @@ module.exports = ProjectEntityUpdateHandler = self =
if not SafePath.isCleanFilename fileName
return callback new Errors.InvalidNameError("invalid element name")
fileRef = new File name : fileName
fileRef = new File(
name: fileName
linkedFileData: linkedFileData
)
FileStoreHandler.uploadFileFromDisk project_id, fileRef._id, fsPath, (err, fileStoreUrl)->
if err?
logger.err err:err, project_id: project_id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
@ -221,7 +224,7 @@ module.exports = ProjectEntityUpdateHandler = self =
return callback(err) if err?
callback null, doc, !existingDoc?
upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, userId, callback = (err, file, isNewFile)->)->
upsertFile: wrapWithLock (project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback = (err, file, isNewFile)->)->
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
@ -231,11 +234,11 @@ module.exports = ProjectEntityUpdateHandler = self =
existingFile = fileRef
break
if existingFile?
self.replaceFile.withoutLock project_id, existingFile._id, fsPath, userId, (err) ->
self.replaceFile.withoutLock project_id, existingFile._id, fsPath, linkedFileData, userId, (err) ->
return callback(err) if err?
callback null, existingFile, !existingFile?
else
self.addFile.withoutLock project_id, folder_id, fileName, fsPath, userId, (err, file) ->
self.addFile.withoutLock project_id, folder_id, fileName, fsPath, linkedFileData, userId, (err, file) ->
return callback(err) if err?
callback null, file, !existingFile?
@ -248,12 +251,12 @@ module.exports = ProjectEntityUpdateHandler = self =
return callback(err) if err?
callback null, doc, isNewDoc, newFolders, folder
upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, userId, callback) ->
upsertFileWithPath: wrapWithLock (project_id, elementPath, fsPath, linkedFileData, userId, callback) ->
fileName = path.basename(elementPath)
folderPath = path.dirname(elementPath)
self.mkdirp.withoutLock project_id, folderPath, (err, newFolders, folder) ->
return callback(err) if err?
self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, userId, (err, file, isNewFile) ->
self.upsertFile.withoutLock project_id, folder._id, fileName, fsPath, linkedFileData, userId, (err, file, isNewFile) ->
return callback(err) if err?
callback null, file, isNewFile, newFolders, folder

26
services/web/app/coffee/Features/ThirdPartyDataStore/UpdateMerger.coffee
View file

@ -1,15 +1,14 @@
_ = require('underscore')
fs = require('fs')
logger = require('logger-sharelatex')
uuid = require('uuid')
EditorController = require('../Editor/EditorController')
FileTypeManager = require('../Uploads/FileTypeManager')
Settings = require('settings-sharelatex')
FileWriter = require('../../infrastructure/FileWriter')
module.exports = UpdateMerger =
mergeUpdate: (user_id, project_id, path, updateRequest, source, callback = (error) ->)->
logger.log project_id:project_id, path:path, "merging update from tpds"
UpdateMerger.p.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
FileWriter.writeStreamToDisk project_id, updateRequest, (err, fsPath)->
return callback(err) if err?
UpdateMerger._mergeUpdate user_id, project_id, path, fsPath, source, (mergeErr) ->
fs.unlink fsPath, (deleteErr) ->
@ -44,29 +43,10 @@ module.exports = UpdateMerger =
processFile: (project_id, fsPath, path, source, user_id, callback)->
logger.log project_id:project_id, "processing file update from tpds"
EditorController.upsertFileWithPath project_id, path, fsPath, source, user_id, (err) ->
EditorController.upsertFileWithPath project_id, path, fsPath, null, source, user_id, (err) ->
logger.log project_id:project_id, "completed processing file update from tpds"
callback(err)
writeStreamToDisk: (project_id, stream, callback = (err, fsPath)->)->
dumpPath = "#{Settings.path.dumpFolder}/#{project_id}_#{uuid.v4()}"
writeStream = fs.createWriteStream(dumpPath)
stream.pipe(writeStream)
stream.on 'error', (err)->
logger.err {err, project_id, dumpPath},
"something went wrong with incoming tpds update stream"
writeStream.on 'error', (err)->
logger.err {err, project_id, dumpPath},
"something went wrong with writing tpds update to disk"
stream.on 'end', ->
logger.log {project_id, dumpPath}, "incoming tpds update stream ended"
writeStream.on "finish", ->
logger.log {project_id, dumpPath}, "tpds update write stream finished"
callback null, dumpPath
readFileIntoTextArray: (path, callback)->
fs.readFile path, "utf8", (error, content = "") ->
if error?

4
services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee
View file

@ -27,9 +27,9 @@ module.exports = FileSystemImportManager =
return callback("path is symlink")
if replace
EditorController.upsertFile project_id, folder_id, name, path, "upload", user_id, callback
EditorController.upsertFile project_id, folder_id, name, path, null, "upload", user_id, callback
else
EditorController.addFile project_id, folder_id, name, path, "upload", user_id, callback
EditorController.addFile project_id, folder_id, name, path, null, "upload", user_id, callback
addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->

23
services/web/app/coffee/infrastructure/FileWriter.coffee Normal file
View file

@ -0,0 +1,23 @@
fs = require 'fs'
logger = require 'logger-sharelatex'
uuid = require 'uuid'
_ = require 'underscore'
Settings = require 'settings-sharelatex'
module.exports =
writeStreamToDisk: (identifier, stream, callback = (error, fsPath) ->) ->
callback = _.once(callback)
fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}"
writeStream = fs.createWriteStream(fsPath)
stream.pipe(writeStream)
stream.on 'error', (err)->
logger.err {err, identifier, fsPath}, "[writeStreamToDisk] something went wrong with incoming stream"
callback(err)
writeStream.on 'error', (err)->
logger.err {err, identifier, fsPath}, "[writeStreamToDisk] something went wrong with writing to disk"
callback(err)
writeStream.on "finish", ->
logger.log {identifier, fsPath}, "[writeStreamToDisk] write stream finished"
callback null, fsPath

1
services/web/app/coffee/models/File.coffee
View file

@ -8,6 +8,7 @@ FileSchema = new Schema
name : type:String, default:''
created : type:Date, default: () -> new Date()
rev : {type:Number, default:0}
linkedFileData: { type: Schema.Types.Mixed }
mongoose.model 'File', FileSchema
exports.File = mongoose.model 'File'

2
services/web/app/coffee/router.coffee
View file

@ -46,6 +46,7 @@ AnnouncementsController = require("./Features/Announcements/AnnouncementsControl
MetaController = require('./Features/Metadata/MetaController')
TokenAccessController = require('./Features/TokenAccess/TokenAccessController')
Features = require('./infrastructure/Features')
LinkedFilesRouter = require './Features/LinkedFiles/LinkedFilesRouter'
logger = require("logger-sharelatex")
_ = require("underscore")
@ -77,6 +78,7 @@ module.exports = class Router
RealTimeProxyRouter.apply(webRouter, privateApiRouter)
ContactRouter.apply(webRouter, privateApiRouter)
AnalyticsRouter.apply(webRouter, privateApiRouter, publicApiRouter)
LinkedFilesRouter.apply(webRouter, privateApiRouter, publicApiRouter)
Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter)

94
services/web/test/acceptance/coffee/LinkedFilesTests.coffee Normal file
View file

@ -0,0 +1,94 @@
async = require "async"
expect = require("chai").expect
MockFileStoreApi = require './helpers/MockFileStoreApi'
MockURLSource = require './helpers/MockURLSource'
request = require "./helpers/request"
User = require "./helpers/User"
MockURLSource.app.get "/foo", (req, res, next) =>
res.send('foo foo foo')
MockURLSource.app.get "/bar", (req, res, next) =>
res.send('bar bar bar')
describe "LinkedFiles", ->
before (done) ->
MockURLSource.run (error) =>
return done(error) if error?
@owner = new User()
@owner.login done
describe "creating a URL based linked file", ->
before (done) ->
@owner.createProject "url-linked-files-project", {template: "blank"}, (error, project_id) =>
throw error if error?
@project_id = project_id
@owner.getProject project_id, (error, project) =>
throw error if error?
@project = project
@root_folder_id = project.rootFolder[0]._id.toString()
done()
it "should download the URL and create a file with the contents and linkedFileData", (done) ->
@owner.request.post {
url: "/project/#{@project_id}/linked_file",
json:
provider: 'url'
data: {
url: "http://localhost:6543/foo"
}
parent_folder_id: @root_folder_id
name: 'url-test-file-1'
}, (error, response, body) =>
throw error if error?
expect(response.statusCode).to.equal 204
@owner.getProject @project_id, (error, project) =>
throw error if error?
file = project.rootFolder[0].fileRefs[0]
expect(file.linkedFileData).to.deep.equal({
provider: 'url'
url: "http://localhost:6543/foo"
})
@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
throw error if error?
expect(response.statusCode).to.equal 200
expect(body).to.equal "foo foo foo"
done()
it "should replace and update a URL based linked file", (done) ->
@owner.request.post {
url: "/project/#{@project_id}/linked_file",
json:
provider: 'url'
data: {
url: "http://localhost:6543/foo"
}
parent_folder_id: @root_folder_id
name: 'url-test-file-2'
}, (error, response, body) =>
throw error if error?
expect(response.statusCode).to.equal 204
@owner.request.post {
url: "/project/#{@project_id}/linked_file",
json:
provider: 'url'
data: {
url: "http://localhost:6543/bar"
}
parent_folder_id: @root_folder_id
name: 'url-test-file-2'
}, (error, response, body) =>
throw error if error?
expect(response.statusCode).to.equal 204
@owner.getProject @project_id, (error, project) =>
throw error if error?
file = project.rootFolder[0].fileRefs[1]
expect(file.linkedFileData).to.deep.equal({
provider: 'url'
url: "http://localhost:6543/bar"
})
@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
throw error if error?
expect(response.statusCode).to.equal 200
expect(body).to.equal "bar bar bar"
done()

12
services/web/test/acceptance/coffee/helpers/MockFileStoreApi.coffee
View file

@ -6,14 +6,22 @@ module.exports = MockFileStoreApi =
run: () ->
app.post "/project/:project_id/file/:file_id", (req, res, next) =>
req.on 'data', ->
chunks = []
req.on 'data', (chunk) ->
chunks.push(chunk)
req.on 'end', =>
content = Buffer.concat(chunks).toString()
{project_id, file_id} = req.params
@files[project_id] ?= {}
@files[project_id][file_id] = { content : "test-file-content" }
@files[project_id][file_id] = { content }
res.sendStatus 200
app.get "/project/:project_id/file/:file_id", (req, res, next) =>
{project_id, file_id} = req.params
{ content } = @files[project_id][file_id]
res.send content
app.listen 3009, (error) ->
throw error if error?
.on "error", (error) ->

7
services/web/test/acceptance/coffee/helpers/MockURLSource.coffee Normal file
View file

@ -0,0 +1,7 @@
express = require("express")
app = express()
module.exports = MockUrlSource =
app: app
run: (callback) ->
app.listen 6543, callback