Merge pull request #623 from sharelatex/sk-support-ext-files-in-v1-import

Support ext files in v1 import

services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee

@@ -1,7 +1,9 @@
 AuthenticationController = require '../Authentication/AuthenticationController'
 EditorController = require '../Editor/EditorController'
+ProjectLocator = require '../Project/ProjectLocator'
 Settings = require 'settings-sharelatex'
 logger = require 'logger-sharelatex'
+_ = require 'underscore'
 
 module.exports = LinkedFilesController = {
 	Agents: {
@@ -9,31 +11,91 @@ module.exports = LinkedFilesController = {
 		project_file: require('./ProjectFileAgent')
 	}
 
+	_getAgent: (provider) ->
+		if !LinkedFilesController.Agents.hasOwnProperty(provider)
+			return null
+		unless provider in Settings.enabledLinkedFileTypes
+			return null
+		LinkedFilesController.Agents[provider]
+
+	_getFileById: (project_id, file_id, callback=(err, file)->) ->
+		ProjectLocator.findElement {
+			project_id,
+			element_id: file_id,
+			type: 'file'
+		}, (err, file, path, parentFolder) ->
+			return callback(err) if err?
+			callback(null, file, path, parentFolder)
+
 	createLinkedFile: (req, res, next) ->
 		{project_id} = req.params
 		{name, provider, data, parent_folder_id} = req.body
 		user_id = AuthenticationController.getLoggedInUserId(req)
 		logger.log {project_id, name, provider, data, parent_folder_id, user_id}, 'create linked file request'
 
-		if !LinkedFilesController.Agents.hasOwnProperty(provider)
-			return res.send(400)
-		unless provider in Settings.enabledLinkedFileTypes
-			return res.send(400)
-		Agent = LinkedFilesController.Agents[provider]
+		Agent = LinkedFilesController._getAgent(provider)
+		if !Agent?
+			return res.sendStatus(400)
 
 		linkedFileData = Agent.sanitizeData(data)
 		linkedFileData.provider = provider
+
+		if !Agent.canCreate(linkedFileData)
+			return res.status(403).send('Cannot create linked file')
+
+		LinkedFilesController._doImport(
+			req, res, next, Agent, project_id, user_id,
+			parent_folder_id, name, linkedFileData
+		)
+
+	refreshLinkedFile: (req, res, next) ->
+		{project_id, file_id} = req.params
+		user_id = AuthenticationController.getLoggedInUserId(req)
+		logger.log {project_id, file_id, user_id}, 'refresh linked file request'
+
+		LinkedFilesController._getFileById project_id, file_id, (err, file, path, parentFolder) ->
+			return next(err) if err?
+			return res.sendStatus(404) if !file?
+			name = file.name
+			linkedFileData = file.linkedFileData
+			if !linkedFileData? || !linkedFileData?.provider?
+				return res.send(409)
+			provider = linkedFileData.provider
+			parent_folder_id = parentFolder._id
+			Agent = LinkedFilesController._getAgent(provider)
+			if !Agent?
+				return res.sendStatus(400)
+			LinkedFilesController._doImport(
+				req, res, next, Agent, project_id, user_id,
+				parent_folder_id, name, linkedFileData
+			)
+
+	_doImport: (req, res, next, Agent, project_id, user_id, parent_folder_id, name, linkedFileData) ->
 		Agent.checkAuth project_id, linkedFileData, user_id, (err, allowed) ->
 			return Agent.handleError(err, req, res, next) if err?
 			return res.sendStatus(403) if !allowed
 			Agent.decorateLinkedFileData linkedFileData, (err, newLinkedFileData) ->
 				return Agent.handleError(err) if err?
 				linkedFileData = newLinkedFileData
-				Agent.writeIncomingFileToDisk project_id, linkedFileData, user_id, (error, fsPath) ->
-					if error?
-						logger.error {err: error, project_id, name, linkedFileData, parent_folder_id, user_id}, 'error writing linked file to disk'
-						return Agent.handleError(error, req, res, next)
-					EditorController.upsertFile project_id, parent_folder_id, name, fsPath, linkedFileData, "upload", user_id, (error, file) ->
-						return next(error) if error?
-						res.json(new_file_id: file._id) # created
+				Agent.writeIncomingFileToDisk project_id,
+					linkedFileData,
+					user_id,
+					(error, fsPath) ->
+						if error?
+							logger.error(
+								{err: error, project_id, name, linkedFileData, parent_folder_id, user_id},
+								'error writing linked file to disk'
+							)
+							return Agent.handleError(error, req, res, next)
+						EditorController.upsertFile project_id,
+							parent_folder_id,
+							name,
+							fsPath,
+							linkedFileData,
+							"upload",
+							user_id,
+							(error, file) ->
+								return next(error) if error?
+								res.json(new_file_id: file._id) # created
+
 	}

services/web/app/coffee/Features/LinkedFiles/LinkedFilesRouter.coffee

@@ -9,3 +9,7 @@ module.exports =
 			AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
 			LinkedFilesController.createLinkedFile
 
+		webRouter.post '/project/:project_id/linked_file/:file_id/refresh',
+			AuthenticationController.requireLogin(),
+			AuthorizationMiddlewear.ensureUserCanWriteProjectContent,
+			LinkedFilesController.refreshLinkedFile

services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee

@@ -2,6 +2,7 @@ FileWriter = require('../../infrastructure/FileWriter')
 AuthorizationManager = require('../Authorization/AuthorizationManager')
 ProjectLocator = require('../Project/ProjectLocator')
 ProjectGetter = require('../Project/ProjectGetter')
+Project = require("../../models/Project").Project
 DocstoreManager = require('../Docstore/DocstoreManager')
 FileStoreHandler = require('../FileStore/FileStoreHandler')
 FileWriter = require('../../infrastructure/FileWriter')
@@ -41,9 +42,17 @@ ProjectNotFoundError = (message) ->
 ProjectNotFoundError.prototype.__proto__ = Error.prototype
 
 
+V1ProjectNotFoundError = (message) ->
+	error = new Error(message)
+	error.name = 'V1ProjectNotFound'
+	error.__proto__ = V1ProjectNotFoundError.prototype
+	return error
+V1ProjectNotFoundError.prototype.__proto__ = Error.prototype
+
+
 SourceFileNotFoundError = (message) ->
 	error = new Error(message)
-	error.name = 'BadData'
+	error.name = 'SourceFileNotFound'
 	error.__proto__ = SourceFileNotFoundError.prototype
 	return error
 SourceFileNotFoundError.prototype.__proto__ = Error.prototype
@@ -55,48 +64,71 @@ module.exports = ProjectFileAgent =
 		return _.pick(
 			data,
 			'source_project_id',
+			'v1_source_doc_id',
 			'source_entity_path'
 		)
 
 	_validate: (data) ->
 		return (
-			data.source_project_id? &&
+			(data.source_project_id? || data.v1_source_doc_id?) &&
 			data.source_entity_path?
 		)
 
+	canCreate: (data) ->
+		# Don't allow creation of linked-files with v1 doc ids
+		!data.v1_source_doc_id?
+
+	_getSourceProject: (data, callback=(err, project)->) ->
+		projection = {_id: 1, name: 1}
+		if data.v1_source_doc_id?
+			Project.findOne {'overleaf.id': data.v1_source_doc_id}, projection, (err, project) ->
+				return callback(err) if err?
+				if !project?
+					return callback(new V1ProjectNotFoundError())
+				callback(null, project)
+		else if data.source_project_id?
+			ProjectGetter.getProject data.source_project_id, projection, (err, project) ->
+				return callback(err) if err?
+				if !project?
+					return callback(new ProjectNotFoundError())
+				callback(null, project)
+		else
+			callback(new BadDataError('neither v1 nor v2 id present'))
+
 	decorateLinkedFileData: (data, callback = (err, newData) ->) ->
 		callback = _.once(callback)
-		{ source_project_id } = data
-		return callback(new BadDataError()) if !source_project_id?
-		ProjectGetter.getProject source_project_id, (err, project) ->
+		@_getSourceProject data, (err, project) ->
 			return callback(err) if err?
-			return callback(new ProjectNotFoundError()) if !project?
 			callback(err, _.extend(data, {source_project_display_name: project.name}))
 
 	checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
 		callback = _.once(callback)
 		if !ProjectFileAgent._validate(data)
 			return callback(new BadDataError())
-		{source_project_id, source_entity_path} = data
-		AuthorizationManager.canUserReadProject current_user_id, source_project_id, null, (err, canRead) ->
+		@_getSourceProject data, (err, project) ->
 			return callback(err) if err?
-			callback(null, canRead)
+			AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) ->
+				return callback(err) if err?
+				callback(null, canRead)
 
 	writeIncomingFileToDisk:
 		(project_id, data, current_user_id, callback = (error, fsPath) ->) ->
 			callback = _.once(callback)
 			if !ProjectFileAgent._validate(data)
 				return callback(new BadDataError())
-			{source_project_id, source_entity_path} = data
-			ProjectLocator.findElementByPath {
-				project_id: source_project_id,
-				path: source_entity_path
-			}, (err, entity, type) ->
-				if err?
-					if err.toString().match(/^not found.*/)
-						err = new SourceFileNotFoundError()
-					return callback(err)
-				ProjectFileAgent._writeEntityToDisk source_project_id, entity._id, type, callback
+			{ source_entity_path } = data
+			@_getSourceProject data, (err, project) ->
+				return callback(err) if err?
+				source_project_id = project._id
+				ProjectLocator.findElementByPath {
+					project_id: source_project_id,
+					path: source_entity_path
+				}, (err, entity, type) ->
+					if err?
+						if err.toString().match(/^not found.*/)
+							err = new SourceFileNotFoundError()
+						return callback(err)
+					ProjectFileAgent._writeEntityToDisk source_project_id, entity._id, type, callback
 
 	_writeEntityToDisk: (project_id, entity_id, type, callback=(err, location)->) ->
 		callback = _.once(callback)
@@ -122,6 +154,8 @@ module.exports = ProjectFileAgent =
 			res.status(404).send("Source file not found")
 		else if error instanceof ProjectNotFoundError
 			res.status(404).send("Project not found")
+		else if error instanceof V1ProjectNotFoundError
+			res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file")
 		else
 			next(error)
 		next()

services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee

@@ -27,6 +27,8 @@ module.exports = UrlAgent = {
 			url: @._prependHttpIfNeeded(data.url)
 		}
 
+	canCreate: (data) -> true
+
 	decorateLinkedFileData: (data, callback = (err, newData) ->) ->
 		return callback(null, data)
 

services/web/app/coffee/Features/Project/ProjectEditorHandler.coffee

@@ -1,5 +1,6 @@
 _ = require("underscore")
 
+
 module.exports = ProjectEditorHandler =
 	trackChangesAvailable: false
 

services/web/app/views/project/editor/binary-file.pug

@@ -52,7 +52,10 @@ div.binary-file.full-size(
 				i.fa.fa-fw.fa-external-link-square.fa-rotate-180.linked-file-icon
 				| Imported from
 				|
-				a(ng-href='/project/{{openFile.linkedFileData.source_project_id}}' target="_blank")
+				a(ng-if='!openFile.linkedFileData.v1_source_doc_id'
+					ng-href='/project/{{openFile.linkedFileData.source_project_id}}' target="_blank")
+					| {{ openFile.linkedFileData.source_project_display_name }}
+				span(ng-if='openFile.linkedFileData.v1_source_doc_id')
 					| {{ openFile.linkedFileData.source_project_display_name }}
 				| /{{ openFile.linkedFileData.source_entity_path.slice(1) }},
 				|

services/web/public/coffee/ide/file-tree/FileTreeManager.coffee

@@ -390,14 +390,11 @@ define [
 
 		refreshLinkedFile: (file) ->
 			parent_folder = @_findParentFolder(file)
-			data = file.linkedFileData
-			provider = data?.provider
-			return if !provider?
-			return @ide.$http.post "/project/#{@ide.project_id}/linked_file", {
-				name: file.name,
-				parent_folder_id: parent_folder?.id
-				provider,
-				data,
+			provider = file.linkedFileData?.provider
+			if !provider?
+				console.warn ">> no provider for #{file.name}", file
+				return
+			return @ide.$http.post "/project/#{@ide.project_id}/linked_file/#{file.id}/refresh", {
 				_csrf: window.csrfToken
 			}
 

services/web/test/acceptance/coffee/LinkedFilesTests.coffee

@@ -94,7 +94,6 @@ describe "LinkedFiles", ->
 				}
 				done()
 
-
 		it 'should import a file from the source project', (done) ->
 			@owner.request.post {
 				url: "/project/#{@project_one_id}/linked_file",
@@ -124,18 +123,13 @@ describe "LinkedFiles", ->
 
 		it 'should refresh the file', (done) ->
 			@owner.request.post {
-				url: "/project/#{@project_one_id}/linked_file",
-				json:
-					name: 'test-link.txt',
-					parent_folder_id: @project_one_root_folder_id,
-					provider: 'project_file',
-					data:
-						source_project_id: @project_two_id,
-						source_entity_path: "/#{@source_doc_name}",
+				url: "/project/#{@project_one_id}/linked_file/#{@existing_file_id}/refresh",
+				json: true
 			}, (error, response, body) =>
 				new_file_id = body.new_file_id
 				expect(new_file_id).to.exist
 				expect(new_file_id).to.not.equal @existing_file_id
+				@refreshed_file_id = new_file_id
 				@owner.getProject @project_one_id, (error, project) =>
 					return done(error) if error?
 					firstFile = project.rootFolder[0].fileRefs[0]
@@ -143,6 +137,55 @@ describe "LinkedFiles", ->
 					expect(firstFile.name).to.equal('test-link.txt')
 					done()
 
+		it 'should not allow to create a linked-file with v1 id', (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_one_id}/linked_file",
+				json:
+					name: 'test-link-should-not-work.txt',
+					parent_folder_id: @project_one_root_folder_id,
+					provider: 'project_file',
+					data:
+						v1_source_doc_id: 1234
+						source_entity_path: "/#{@source_doc_name}",
+			}, (error, response, body) =>
+				expect(response.statusCode).to.equal 403
+				expect(body).to.equal 'Cannot create linked file'
+				done()
+
+	describe "with a linked project_file from a v1 project that has not been imported", ->
+		before (done) ->
+			async.series [
+				(cb) =>
+					@owner.createProject 'plf-v1-test-one', {template: 'blank'}, (error, project_id) =>
+						@project_one_id = project_id
+						cb(error)
+				(cb) =>
+					@owner.getProject @project_one_id, (error, project) =>
+						@project_one = project
+						@project_one_root_folder_id = project.rootFolder[0]._id.toString()
+						@project_one.rootFolder[0].fileRefs.push {
+							linkedFileData: {
+								provider: "project_file",
+								v1_source_doc_id: 9999999,  # We won't find this id in the database
+								source_entity_path: "example.jpeg"
+							},
+							_id: "abcd",
+							rev: 0,
+							created: new Date(),
+							name: "example.jpeg"
+						}
+						@owner.saveProject @project_one, cb
+			], done
+
+		it 'should refuse to refresh', (done) ->
+			@owner.request.post {
+				url: "/project/#{@project_one_id}/linked_file/abcd/refresh",
+				json: true
+			}, (error, response, body) =>
+				expect(response.statusCode).to.equal 409
+				expect(body).to.equal "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file"
+				done()
+
 	describe "creating a URL based linked file", ->
 		before (done) ->
 			@owner.createProject "url-linked-files-project", {template: "blank"}, (error, project_id) =>

services/web/test/acceptance/coffee/helpers/User.coffee

@@ -145,6 +145,9 @@ class User
 	getProject: (project_id, callback = (error, project)->) ->
 		db.projects.findOne {_id: ObjectId(project_id.toString())}, callback
 
+	saveProject: (project, callback=(error)->) ->
+		db.projects.update {_id: project._id}, project, callback
+
 	createProject: (name, options, callback = (error, oroject_id) ->) ->
 		if typeof options == "function"
 			callback = options

services/web/test/unit/coffee/Editor/EditorRealTimeControllerTests.coffee

@@ -8,11 +8,11 @@ describe "EditorRealTimeController", ->
 		@rclient =
 			publish: sinon.stub()
 		@EditorRealTimeController = SandboxedModule.require modulePath, requires:
-			"../../infrastructure/RedisWrapper": 
+			"../../infrastructure/RedisWrapper":
 				client: () => @rclient
 			"../../infrastructure/Server" : io: @io = {}
 			"settings-sharelatex":{redis:{}}
-		
+
 		@room_id = "room-id"
 		@message = "message-to-editor"
 		@payload = ["argument one", 42]