Merge pull request #436 from sharelatex/mm-project-export

Project exports spike

services/web/README.md

@@ -3,11 +3,11 @@ web-sharelatex
 
 web-sharelatex is the front-end web service of the open-source web-based collaborative LaTeX editor,
 [ShareLaTeX](https://www.sharelatex.com).
-It serves all the HTML pages, CSS and javascript to the client. web-sharelatex also contains 
+It serves all the HTML pages, CSS and javascript to the client. web-sharelatex also contains
 a lot of logic around creating and editing projects, and account management.
 
 
-The rest of the ShareLaTeX stack, along with information about contributing can be found in the 
+The rest of the ShareLaTeX stack, along with information about contributing can be found in the
 [sharelatex/sharelatex](https://github.com/sharelatex/sharelatex) repository.
 
 Build process
@@ -20,7 +20,7 @@ Image processing tasks are commented out in the gruntfile and the needed package
 New Docker-based build process
 ------------------------------
 
-Note that the Grunt workflow from above should still work, but we are transitioning to a 
+Note that the Grunt workflow from above should still work, but we are transitioning to a
 Docker based testing workflow, which is documented below:
 
 ### Running the app
@@ -59,19 +59,18 @@ Acceptance tests are run against a live service, which runs in the `acceptance_t
 To run the tests out-of-the-box, the makefile defines:
 
 ```
-make install # Only needs running once, or when npm packages are updated
-make acceptance_test
+make test_acceptance
 ```
 
 However, during development it is often useful to leave the service running for rapid iteration on the acceptance tests. This can be done with:
 
 ```
-make acceptance_test_start_service
-make acceptance_test_run # Run as many times as needed during development
-make acceptance_test_stop_service
+make test_acceptance_app_start_service
+make test_acceptance_app_run # Run as many times as needed during development
+make test_acceptance_app_stop_service
 ```
 
-`make acceptance_test` just runs these three commands in sequence.
+`make test_acceptance` just runs these three commands in sequence and then runs `make test_acceptance_modules` which performs the tests for each module in the `modules` directory. (Note that there is not currently an equivalent to the `-start` / `-run` x _n_ / `-stop` series for modules.)
 
 During development it is often useful to only run a subset of tests, which can be configured with arguments to the mocha CLI:
 
@@ -111,12 +110,3 @@ We gratefully acknowledge [IconShock](http://www.iconshock.com) for use of the i
 in the `public/img/iconshock` directory found via
 [findicons.com](http://findicons.com/icon/498089/height?id=526085#)
 
-
-## Acceptance Tests
-
-To run the Acceptance tests:
-
-- set `allowPublicAccess` to true, either in the configuration file,
-  or by setting the environment variable `SHARELATEX_ALLOW_PUBLIC_ACCESS` to `true`
-- start the server (`grunt`)
-- in a separate terminal, run `grunt test:acceptance`

services/web/app/coffee/Features/Exports/ExportsController.coffee

@@ -0,0 +1,18 @@
+ExportsHandler = require("./ExportsHandler")
+AuthenticationController = require("../Authentication/AuthenticationController")
+logger = require("logger-sharelatex")
+
+module.exports =
+
+	exportProject: (req, res) ->
+		{project_id, brand_variation_id} = req.params
+		user_id = AuthenticationController.getLoggedInUserId(req)
+		ExportsHandler.exportProject project_id, user_id, brand_variation_id, (err, export_data) ->
+			logger.log 
+				user_id:user_id
+				project_id: project_id
+				brand_variation_id:brand_variation_id
+				export_v1_id:export_data.v1_id
+				"exported project"
+			res.send export_v1_id: export_data.v1_id
+

services/web/app/coffee/Features/Exports/ExportsHandler.coffee

@@ -0,0 +1,93 @@
+ProjectGetter = require('../Project/ProjectGetter')
+ProjectLocator = require('../Project/ProjectLocator')
+UserGetter = require('../User/UserGetter')
+logger = require('logger-sharelatex')
+settings = require 'settings-sharelatex'
+async = require 'async'
+request = require 'request'
+request = request.defaults()
+settings = require 'settings-sharelatex'
+
+module.exports = ExportsHandler = self =
+
+	exportProject: (project_id, user_id, brand_variation_id, callback=(error, export_data) ->) ->
+		self._buildExport project_id, user_id, brand_variation_id, (err, export_data) ->
+			return callback(err) if err?
+			self._requestExport export_data, (err, export_v1_id) ->
+				return callback(err) if err?
+				export_data.v1_id = export_v1_id
+				# TODO: possibly store the export data in Mongo
+				callback null, export_data
+
+	_buildExport: (project_id, user_id, brand_variation_id, callback=(err, export_data) ->) ->
+		jobs =
+			project: (cb) ->
+				ProjectGetter.getProject project_id, cb
+			# TODO: when we update async, signature will change from (cb, results) to (results, cb)
+			rootDoc: [ 'project', (cb, results) ->
+				ProjectLocator.findRootDoc {project: results.project, project_id: project_id}, cb
+			]
+			user: (cb) ->
+				UserGetter.getUser user_id, {first_name: 1, last_name: 1, email: 1}, cb
+			historyVersion: (cb) ->
+				self._requestVersion project_id, cb
+
+		async.auto jobs, (err, results) ->
+			if err?
+				logger.err err:err, project_id:project_id, user_id:user_id, brand_variation_id:brand_variation_id, "error building project export"
+				return callback(err)
+
+			{project, rootDoc, user, historyVersion} = results
+			if !rootDoc[1]?
+				err = new Error("cannot export project without root doc")
+				logger.err err:err, project_id: project_id
+				return callback(err)
+
+			export_data =
+				project:
+					id: project_id
+					rootDocPath: rootDoc[1]?.fileSystem
+					historyId: project.overleaf?.history?.id
+					historyVersion: historyVersion
+				user:
+					id: user_id
+					firstName: user.first_name
+					lastName: user.last_name
+					email: user.email
+					orcidId: null # until v2 gets ORCID
+				destination:
+					brandVariationId: brand_variation_id
+				options:
+					callbackUrl: null # for now, until we want v1 to call us back
+			callback null, export_data
+
+	_requestExport: (export_data, callback=(err, export_v1_id) ->) ->
+		request.post {
+			url: "#{settings.apis.v1.url}/api/v1/sharelatex/exports"
+			auth: {user: settings.apis.v1.user, pass: settings.apis.v1.pass }
+			json: export_data
+		}, (err, res, body) ->
+			if err?
+				logger.err err:err, export:export_data, "error making request to v1 export"
+				callback err
+			else if 200 <= res.statusCode < 300
+				callback null, body.exportId
+			else
+				err = new Error("v1 export returned a failure status code: #{res.statusCode}")
+				logger.err err:err, export:export_data, "v1 export returned failure status code: #{res.statusCode}"
+				callback err
+
+	_requestVersion: (project_id, callback=(err, export_v1_id) ->) ->
+		request.get {
+			url: "#{settings.apis.project_history.url}/project/#{project_id}/version"
+			json: true
+		}, (err, res, body) ->
+			if err?
+				logger.err err:err, project_id:project_id, "error making request to project history"
+				callback err
+			else if res.statusCode >= 200 and res.statusCode < 300
+				callback null, body.version
+			else
+				err = new Error("project history version returned a failure status code: #{res.statusCode}")
+				logger.err err:err, project_id:project_id, "project history version returned failure status code: #{res.statusCode}"
+				callback err

services/web/app/coffee/router.coffee

@@ -26,6 +26,7 @@ HealthCheckController = require("./Features/HealthCheck/HealthCheckController")
 ProjectDownloadsController = require "./Features/Downloads/ProjectDownloadsController"
 FileStoreController = require("./Features/FileStore/FileStoreController")
 HistoryController = require("./Features/History/HistoryController")
+ExportsController = require("./Features/Exports/ExportsController")
 PasswordResetRouter = require("./Features/PasswordReset/PasswordResetRouter")
 StaticPagesRouter = require("./Features/StaticPages/StaticPagesRouter")
 ChatController = require("./Features/Chat/ChatController")
@@ -205,6 +206,7 @@ module.exports = class Router
 		webRouter.post "/project/:project_id/restore_file", AuthorizationMiddlewear.ensureUserCanWriteProjectContent, HistoryController.restoreFileFromV2
 		privateApiRouter.post "/project/:Project_id/history/resync", AuthenticationController.httpAuth, HistoryController.resyncProjectHistory
 
+		webRouter.post '/project/:project_id/export/:brand_variation_id', AuthorizationMiddlewear.ensureUserCanAdminProject, ExportsController.exportProject
 
 		webRouter.get  '/Project/:Project_id/download/zip', AuthorizationMiddlewear.ensureUserCanReadProject, ProjectDownloadsController.downloadProject
 		webRouter.get  '/project/download/zip', AuthorizationMiddlewear.ensureUserCanReadMultipleProjects, ProjectDownloadsController.downloadMultipleProjects

services/web/config/settings.defaults.coffee

@@ -156,6 +156,10 @@ module.exports = settings =
 			url: process.env['LINKED_URL_PROXY']
 		thirdpartyreferences:
 			url: "http://#{process.env['THIRD_PARTY_REFERENCES_HOST'] or 'localhost'}:3046"
+		v1:
+			url: "http://#{process.env['V1_HOST'] or 'localhost'}:5000"
+			user: 'overleaf'
+			pass: 'password'
 
 	templates:
 		user_id: process.env.TEMPLATES_USER_ID or "5395eb7aad1f29a88756c7f2"
@@ -363,7 +367,7 @@ module.exports = settings =
 
 	appName: "ShareLaTeX (Community Edition)"
 	adminEmail: "placeholder@example.com"
-	
+
 	brandPrefix: "" # Set to 'ol-' for overleaf styles
 
 	nav:

services/web/test/acceptance/coffee/ExportsTests.coffee

@@ -0,0 +1,56 @@
+expect = require('chai').expect
+request = require './helpers/request'
+_ = require 'underscore'
+
+
+User = require './helpers/User'
+ProjectGetter = require '../../../app/js/Features/Project/ProjectGetter.js'
+ExportsHandler = require '../../../app/js/Features/Exports/ExportsHandler.js'
+
+MockProjectHistoryApi = require './helpers/MockProjectHistoryApi'
+MockV1Api = require './helpers/MockV1Api'
+
+describe 'Exports', ->
+	before (done) ->
+		@brand_variation_id = '18'
+		@owner = new User()
+		@owner.login (error) =>
+			throw error if error?
+			@owner.createProject 'example-project', {template: 'example'}, (error, @project_id) =>
+				throw error if error?
+				done()
+
+	describe 'exporting a project', ->
+		beforeEach (done) ->
+			@version = Math.floor(Math.random() * 10000)
+			MockProjectHistoryApi.setProjectVersion(@project_id, @version)
+			@export_id = Math.floor(Math.random() * 10000)
+			MockV1Api.setExportId(@export_id)
+			MockV1Api.clearExportParams()
+			@owner.request {
+				method: 'POST',
+				url: "/project/#{@project_id}/export/#{@brand_variation_id}",
+				json: {},
+			}, (error, response, body) =>
+				throw error if error?
+				expect(response.statusCode).to.equal 200
+				@exportResponseBody = body
+				done()
+
+		it 'should have sent correct data to v1', (done) ->
+			{project, user, destination, options} = MockV1Api.getLastExportParams()
+			# project details should match
+			expect(project.id).to.equal @project_id
+			expect(project.rootDocPath).to.equal '/main.tex'
+			# version should match what was retrieved from project-history
+			expect(project.historyVersion).to.equal @version
+			# user details should match
+			expect(user.id).to.equal @owner.id
+			expect(user.email).to.equal @owner.email
+			# brand-variation should match
+			expect(destination.brandVariationId).to.equal @brand_variation_id
+			done()
+
+		it 'should have returned the export ID provided by v1', (done) ->
+			expect(@exportResponseBody.export_v1_id).to.equal @export_id
+			done()

services/web/test/acceptance/coffee/helpers/MockProjectHistoryApi.coffee

@@ -6,9 +6,14 @@ module.exports = MockProjectHistoryApi =
 
 	oldFiles: {}
 
+	projectVersions: {}
+
 	addOldFile: (project_id, version, pathname, content) ->
 		@oldFiles["#{project_id}:#{version}:#{pathname}"] = content
 
+	setProjectVersion: (project_id, version) ->
+		@projectVersions[project_id] = version
+
 	run: () ->
 		app.post "/project", (req, res, next) =>
 			res.json project: id: 1
@@ -21,6 +26,13 @@ module.exports = MockProjectHistoryApi =
 			else
 				res.send 404
 
+		app.get "/project/:project_id/version", (req, res, next) =>
+			{project_id} = req.params
+			if @projectVersions[project_id]?
+				res.json version: @projectVersions[project_id]
+			else
+				res.send 404
+
 		app.listen 3054, (error) ->
 			throw error if error?
 		.on "error", (error) ->

services/web/test/acceptance/coffee/helpers/MockV1Api.coffee

@@ -0,0 +1,34 @@
+express = require("express")
+app = express()
+bodyParser = require('body-parser')
+
+app.use(bodyParser.json())
+
+module.exports = MockV1Api =
+
+	exportId: null
+
+	exportParams: null
+
+	setExportId: (id) ->
+		@exportId = id
+
+	getLastExportParams: () ->
+		@exportParams
+
+	clearExportParams: () ->
+		@exportParams = null
+
+	run: () ->
+		app.post "/api/v1/sharelatex/exports", (req, res, next) =>
+			#{project, version, pathname}
+			@exportParams = Object.assign({}, req.body)
+			res.json exportId: @exportId
+
+		app.listen 5000, (error) ->
+			throw error if error?
+		.on "error", (error) ->
+			console.error "error starting MockOverleafAPI:", error.message
+			process.exit(1)
+
+MockV1Api.run()

services/web/test/unit/coffee/Exports/ExportsControllerTests.coffee

@@ -0,0 +1,39 @@
+SandboxedModule = require('sandboxed-module')
+assert = require('assert')
+chai = require('chai')
+expect = chai.expect
+sinon = require('sinon')
+modulePath = require('path').join __dirname, '../../../../app/js/Features/Exports/ExportsController.js'
+
+
+describe 'ExportsController', ->
+	project_id = "123njdskj9jlk"
+	user_id = "123nd3ijdks"
+	brand_variation_id = 22
+
+	beforeEach ->
+		@handler =
+			getUserNotifications: sinon.stub().callsArgWith(1)
+		@req =
+			params:
+				project_id: project_id
+				brand_variation_id: brand_variation_id
+			session:
+				user:
+					_id:user_id
+			i18n:
+				translate:->
+		@AuthenticationController =
+			getLoggedInUserId: sinon.stub().returns(@req.session.user._id)
+		@controller = SandboxedModule.require modulePath, requires:
+			"./ExportsHandler":@handler
+			'logger-sharelatex':
+				log:->
+				err:->
+			'../Authentication/AuthenticationController': @AuthenticationController
+
+	it 'should ask the handler to perform the export', (done) ->
+		@handler.exportProject = sinon.stub().yields(null, {iAmAnExport: true, v1_id: 897})
+		@controller.exportProject @req, send:(body) =>
+			expect(body).to.deep.equal {export_v1_id: 897}
+			done()

services/web/test/unit/coffee/Exports/ExportsHandlerTests.coffee

@@ -0,0 +1,202 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = '../../../../app/js/Features/Exports/ExportsHandler.js'
+SandboxedModule = require('sandboxed-module')
+
+describe 'ExportsHandler', ->
+
+	beforeEach ->
+		@ProjectGetter = {}
+		@ProjectLocator = {}
+		@UserGetter = {}
+		@settings = {}
+		@stubRequest = {}
+		@request = defaults: => return @stubRequest
+		@ExportsHandler = SandboxedModule.require modulePath, requires:
+			'logger-sharelatex':
+				log: ->
+				err: ->
+			'../Project/ProjectGetter': @ProjectGetter
+			'../Project/ProjectLocator': @ProjectLocator
+			'../User/UserGetter': @UserGetter
+			'settings-sharelatex': @settings
+			'request': @request
+		@project_id = "project-id-123"
+		@project_history_id = 987
+		@user_id = "user-id-456"
+		@brand_variation_id = 789
+		@callback = sinon.stub()
+
+	describe 'exportProject', ->
+		beforeEach (done) ->
+			@export_data = {iAmAnExport: true}
+			@response_body = {iAmAResponseBody: true}
+			@ExportsHandler._buildExport = sinon.stub().yields(null, @export_data)
+			@ExportsHandler._requestExport = sinon.stub().yields(null, @response_body)
+			@ExportsHandler.exportProject @project_id, @user_id, @brand_variation_id, (error, export_data) =>
+				@callback(error, export_data)
+				done()
+
+		it "should build the export", ->
+			@ExportsHandler._buildExport
+			.calledWith(@project_id, @user_id, @brand_variation_id)
+			.should.equal true
+
+		it "should request the export", ->
+			@ExportsHandler._requestExport
+			.calledWith(@export_data)
+			.should.equal true
+
+		it "should return the export", ->
+			@callback
+			.calledWith(null, @export_data)
+			.should.equal true
+
+	describe '_buildExport', ->
+		beforeEach (done) ->
+			@project =
+				id: @project_id
+				overleaf:
+					history:
+						id: @project_history_id
+			@user =
+				id: @user_id
+				first_name: 'Arthur'
+				last_name: 'Author'
+				email: 'arthur.author@arthurauthoring.org'
+			@rootDocPath = 'main.tex'
+			@historyVersion = 777
+			@ProjectGetter.getProject = sinon.stub().yields(null, @project)
+			@ProjectLocator.findRootDoc = sinon.stub().yields(null, [null, {fileSystem: 'main.tex'}])
+			@UserGetter.getUser = sinon.stub().yields(null, @user)
+			@ExportsHandler._requestVersion = sinon.stub().yields(null, @historyVersion)
+			done()
+
+		describe "when all goes well", ->
+			beforeEach (done) ->
+				@ExportsHandler._buildExport @project_id, @user_id, @brand_variation_id, (error, export_data) =>
+					@callback(error, export_data)
+					done()
+
+			it "should request the project history version", ->
+				@ExportsHandler._requestVersion.called
+				.should.equal true
+
+			it "should return export data", ->
+				expected_export_data = 
+					project:
+						id: @project_id
+						rootDocPath: @rootDocPath
+						historyId: @project_history_id
+						historyVersion: @historyVersion
+					user:
+						id: @user_id
+						firstName: @user.first_name
+						lastName: @user.last_name
+						email: @user.email
+						orcidId: null
+					destination:
+						brandVariationId: @brand_variation_id
+					options:
+						callbackUrl: null
+				@callback.calledWith(null, expected_export_data)
+				.should.equal true
+
+		describe "when project is not found", ->
+			beforeEach (done) ->
+				@ProjectGetter.getProject = sinon.stub().yields(new Error("project not found"))
+				@ExportsHandler._buildExport @project_id, @user_id, @brand_variation_id, (error, export_data) =>
+					@callback(error, export_data)
+					done()
+
+			it "should return an error", ->
+				(@callback.args[0][0] instanceof Error)
+				.should.equal true
+
+		describe "when project has no root doc", ->
+			beforeEach (done) ->
+				@ProjectLocator.findRootDoc = sinon.stub().yields(null, [null, null])
+				@ExportsHandler._buildExport @project_id, @user_id, @brand_variation_id, (error, export_data) =>
+					@callback(error, export_data)
+					done()
+
+			it "should return an error", ->
+				(@callback.args[0][0] instanceof Error)
+				.should.equal true
+
+		describe "when user is not found", ->
+			beforeEach (done) ->
+				@UserGetter.getUser = sinon.stub().yields(new Error("user not found"))
+				@ExportsHandler._buildExport @project_id, @user_id, @brand_variation_id, (error, export_data) =>
+					@callback(error, export_data)
+					done()
+
+			it "should return an error", ->
+				(@callback.args[0][0] instanceof Error)
+				.should.equal true
+
+		describe "when project history request fails", ->
+			beforeEach (done) ->
+				@ExportsHandler._requestVersion = sinon.stub().yields(new Error("project history call failed"))
+				@ExportsHandler._buildExport @project_id, @user_id, @brand_variation_id, (error, export_data) =>
+					@callback(error, export_data)
+					done()
+
+			it "should return an error", ->
+				(@callback.args[0][0] instanceof Error)
+				.should.equal true
+
+	describe '_requestExport', ->
+		beforeEach (done) ->
+			@settings.apis =
+				v1:
+					url: 'http://localhost:5000'
+					user: 'overleaf'
+					pass: 'pass'
+			@export_data = {iAmAnExport: true}
+			@export_id = 4096
+			@stubPost = sinon.stub().yields(null, {statusCode: 200}, { exportId: @export_id })
+			done()
+
+		describe "when all goes well", ->
+			beforeEach (done) ->
+				@stubRequest.post = @stubPost
+				@ExportsHandler._requestExport @export_data, (error, export_v1_id) =>
+					@callback(error, export_v1_id)
+					done()
+
+			it 'should issue the request', ->
+				expect(@stubPost.getCall(0).args[0]).to.deep.equal
+					url: @settings.apis.v1.url + '/api/v1/sharelatex/exports'
+					auth:
+						user: @settings.apis.v1.user
+						pass: @settings.apis.v1.pass
+					json: @export_data
+
+			it 'should return the v1 export id', ->
+				@callback.calledWith(null, @export_id)
+				.should.equal true
+
+		describe "when the request fails", ->
+			beforeEach (done) ->
+				@stubRequest.post = sinon.stub().yields(new Error("export request failed"))
+				@ExportsHandler._requestExport @export_data, (error, export_v1_id) =>
+					@callback(error, export_v1_id)
+					done()
+
+			it "should return an error", ->
+				(@callback.args[0][0] instanceof Error)
+				.should.equal true
+
+		describe "when the request returns an error code", ->
+			beforeEach (done) ->
+				@stubRequest.post = sinon.stub().yields(null, {statusCode: 401}, { })
+				@ExportsHandler._requestExport @export_data, (error, export_v1_id) =>
+					@callback(error, export_v1_id)
+					done()
+
+			it "should return the error", ->
+				(@callback.args[0][0] instanceof Error)
+				.should.equal true