From 1f20744a68d2ca80f204721310affdbd5d4c5999 Mon Sep 17 00:00:00 2001 From: Henrique Santos Date: Sat, 12 Mar 2016 11:08:07 -0300 Subject: [PATCH] add size function for each persitor --- services/filestore/app.coffee | 3 ++- .../app/coffee/AWSSDKPersistorManager.coffee | 12 +++++++++++- .../app/coffee/FSPersistorManager.coffee | 16 ++++++++++++++-- .../app/coffee/ProjectController.coffee | 4 ++-- .../filestore/app/coffee/ProjectHandler.coffee | 8 ++++---- .../app/coffee/S3PersistorManager.coffee | 14 +++++++++++++- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee index 93e86164a7..ad05bd5124 100644 --- a/services/filestore/app.coffee +++ b/services/filestore/app.coffee @@ -4,6 +4,7 @@ logger.initialize("filestore") settings = require("settings-sharelatex") request = require("request") fileController = require("./app/js/FileController") +projectController = require("./app/js/ProjectController") keyBuilder = require("./app/js/KeyBuilder") domain = require("domain") appIsOk = true @@ -85,7 +86,7 @@ app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, express.bodyParser(), fileController.copyFile app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile -app.get "project/:project_id/size", keyBuilder.publicProjectKey, projectController.projectSize +app.get "/project/:project_id/size", keyBuilder.publicProjectKey, projectController.projectSize app.get "/heapdump", (req, res)-> require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee index 0c5cb959f9..ca72901f8f 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.coffee @@ -87,5 +87,15 @@ module.exports = getProjectSize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" - callback null, 1024 + s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" + return callback err + if data.Contents.length == 0 + logger.log bucketName:bucketName, key:key, "the directory is empty" + return callback() + totalSize = 0 + _.each data.Contents, (entry)-> + totalSize += entry.Size + callback null, totalSize diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 17f11217c1..c146b162ff 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -89,5 +89,17 @@ module.exports = callback null, exists getProjectSize:(location, name, callback)-> - logger.log location:location, name:name, "get project size in file system" - callback null, 1024 + filteredName = filterName name.replace(/\/$/,'') + logger.log location:location, name:filteredName, "get project size in file system" + fs.readdir "#{location}/#{filteredName}", (err, files) -> + if err? + logger.err err:err, location:location, name:filteredName, "something went wrong listing prefix in aws" + return callback(err) + totalSize = 0 + _.each files, (entry)-> + fd = fs.openSync "#{location}/#{filteredName}/#{entry}", 'r' + fileStats = fs.fstatSync(fd) + totalSize += fileStats.size + fs.closeSync fd + logger.log totalSize:totalSize, "total size", files:files + callback null, totalSize diff --git a/services/filestore/app/coffee/ProjectController.coffee b/services/filestore/app/coffee/ProjectController.coffee index 21f30f117b..e31d2539b5 100644 --- a/services/filestore/app/coffee/ProjectController.coffee +++ b/services/filestore/app/coffee/ProjectController.coffee @@ -2,7 +2,7 @@ settings = require("settings-sharelatex") logger = require("logger-sharelatex") metrics = require("metrics-sharelatex") Errors = require('./Errors') - +ProjectHandler = require("./ProjectHandler") module.exports = projectController = @@ -10,7 +10,7 @@ module.exports = projectController = metrics.inc "projectSize" {project_id, bucket} = req logger.log project_id:project_id, bucket:bucket, "reciving request to project size" - ProjectHandler.getSize bucket, project_id, req, (err, size)-> + ProjectHandler.getSize bucket, project_id, (err, size)-> if err? logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" res.send 500 diff --git a/services/filestore/app/coffee/ProjectHandler.coffee b/services/filestore/app/coffee/ProjectHandler.coffee index e6022c4aa3..53ed2df183 100644 --- a/services/filestore/app/coffee/ProjectHandler.coffee +++ b/services/filestore/app/coffee/ProjectHandler.coffee @@ -5,9 +5,9 @@ async = require("async") module.exports = - getSize: (bucket, project_id, opts = {}, callback)-> - logger.log bucket:bucket, project_id:project_id, opts:opts, "getting project size" - PersistorManager.getProjectSize bucket, project_id, opts, (err, size)-> + getSize: (bucket, project_id, callback)-> + logger.log bucket:bucket, project_id:project_id, "getting project size" + PersistorManager.getProjectSize bucket, project_id, (err, size)-> if err? - logger.err bucket:bucket, project_id:project_id, opts:opts, "error getting size" + logger.err bucket:bucket, project_id:project_id, "error getting size" callback err, size \ No newline at end of file diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index b6380eceb8..4accd517cb 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -140,4 +140,16 @@ module.exports = getProjectSize:(bucketName, key, callback)-> logger.log bucketName:bucketName, key:key, "get project size in s3" - callback null, 1024 + s3Client = knox.createClient + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret + bucket: bucketName + s3Client.list prefix:key, (err, data)-> + if err? + logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" + return callback(err) + totalSize = 0 + _.each data.Contents, (entry)-> + totalSize += entry.Size + logger.log totalSize:totalSize, "total size" + callback null, totalSize