From 314c7350042848a8a721199899fa5f5a2119e382 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 1 Jun 2015 18:22:02 -0300 Subject: [PATCH] initial version for archive docs in S3 --- services/docstore/app.coffee | 1 + .../docstore/app/coffee/DocManager.coffee | 35 ++++++++++++++++++ .../docstore/app/coffee/HttpController.coffee | 7 ++++ .../docstore/config/settings.defaults.coffee | 8 +++++ .../acceptance/coffee/ArchiveDocsTests.coffee | 36 +++++++++++++++++++ .../coffee/helpers/DocstoreClient.coffee | 14 +++++--- 6 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee diff --git a/services/docstore/app.coffee b/services/docstore/app.coffee index ec9b250851..8609d79e8f 100644 --- a/services/docstore/app.coffee +++ b/services/docstore/app.coffee @@ -20,6 +20,7 @@ app.get '/project/:project_id/doc/:doc_id', HttpController.getDoc app.get '/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc app.post '/project/:project_id/doc/:doc_id', bodyParser.json(limit: "2mb"), HttpController.updateDoc app.del '/project/:project_id/doc/:doc_id', HttpController.deleteDoc +app.get '/project/:project_id/archive', HttpController.archiveAllDocs app.get '/status', (req, res)-> res.send('docstore is alive') diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index ef03969a30..57184d8844 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -3,6 +3,10 @@ Errors = require "./Errors" logger = require "logger-sharelatex" _ = require "underscore" async = require "async" +settings = require("settings-sharelatex") +request = require("request") +crypto = require("crypto") +thirtySeconds = 30 * 1000 module.exports = DocManager = @@ -58,3 +62,34 @@ module.exports = DocManager = return callback(error) if error? callback() + archiveAllDocs: (project_id, callback = (error, docs) ->) -> + MongoManager.getProjectsDocs project_id, (error, docs) -> + if err? + return callback(error) + else if !docs? + return callback new Errors.NotFoundError("No docs for project #{project_id}") + + jobs = for doc in docs + do (doc) => + (cb) => + logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3" + options = buildS3Options(doc.lines, project_id+"/"+doc._id) + request.put options, (err, res)-> + if err? || res.statusCode != 200 + logger.err err:err, res:res, "something went wrong archiving file in aws" + cb(err) + + async.series jobs, callback + +buildS3Options = (content, key)-> + return { + aws: + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: settings.filestore.stores.user_files + timeout: thirtySeconds + json: content + #headers: + # 'content-md5': crypto.createHash("md5").update(content).digest("hex") + uri:"https://#{settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + } diff --git a/services/docstore/app/coffee/HttpController.coffee b/services/docstore/app/coffee/HttpController.coffee index 2a974794b2..1c790ac9ba 100644 --- a/services/docstore/app/coffee/HttpController.coffee +++ b/services/docstore/app/coffee/HttpController.coffee @@ -78,3 +78,10 @@ module.exports = HttpController = _buildRawDocView: (doc)-> return (doc?.lines or []).join("\n") + + archiveAllDocs: (req, res, next = (error) ->) -> + project_id = req.params.project_id + logger.log project_id: project_id, "archiving all docs" + DocManager.archiveAllDocs project_id, (error) -> + return next(error) if error? + res.send 204 diff --git a/services/docstore/config/settings.defaults.coffee b/services/docstore/config/settings.defaults.coffee index 8a3fc9197f..d708b56ce2 100644 --- a/services/docstore/config/settings.defaults.coffee +++ b/services/docstore/config/settings.defaults.coffee @@ -9,3 +9,11 @@ module.exports = mongo: url: 'mongodb://127.0.0.1/sharelatex' + + #filestore: + # backend: "s3" + # stores: + # user_files: "" + # s3: + # key: "" + # secret: "" \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee new file mode 100644 index 0000000000..f55e39099c --- /dev/null +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -0,0 +1,36 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +{ObjectId} = require "mongojs" +async = require "async" + +DocstoreClient = require "./helpers/DocstoreClient" + +describe "Archiving all docs", -> + beforeEach (done) -> + @project_id = ObjectId() + @docs = [{ + _id: ObjectId() + lines: ["one", "two", "three"] + rev: 2 + }, { + _id: ObjectId() + lines: ["aaa", "bbb", "ccc"] + rev: 4 + }, { + _id: ObjectId() + lines: ["111", "222", "333"] + rev: 6 + }] + jobs = for doc in @docs + do (doc) => + (callback) => + DocstoreClient.createDoc @project_id, doc._id, doc.lines, (err)=> + doc.lines[0] = doc.lines[0]+" added" + DocstoreClient.updateDoc @project_id, doc._id, doc.lines, callback + async.series jobs, done + + it "should archive all the docs", (done) -> + DocstoreClient.archiveAllDoc @project_id, (error, res) => + res.statusCode.should.equal 204 + done() \ No newline at end of file diff --git a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee index 1d2bcadc75..989a2ce0e7 100644 --- a/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee +++ b/services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee @@ -1,5 +1,6 @@ request = require("request").defaults(jar: false) {db, ObjectId} = require("../../../../app/js/mongojs") +settings = require("settings-sharelatex") module.exports = DocstoreClient = @@ -16,28 +17,31 @@ module.exports = DocstoreClient = getDoc: (project_id, doc_id, qs, callback = (error, res, body) ->) -> request.get { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" json: true qs:qs }, callback getAllDocs: (project_id, callback = (error, res, body) ->) -> request.get { - url: "http://localhost:3016/project/#{project_id}/doc" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc" json: true }, callback updateDoc: (project_id, doc_id, lines, callback = (error, res, body) ->) -> request.post { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" json: lines: lines }, callback deleteDoc: (project_id, doc_id, callback = (error, res, body) ->) -> request.del { - url: "http://localhost:3016/project/#{project_id}/doc/#{doc_id}" + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/doc/#{doc_id}" }, callback - + archiveAllDoc: (project_id, callback = (error, res, body) ->) -> + request.get { + url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive" + }, callback