github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Prettier: convert app/js decaffeinated files to Prettier format

Browse source
This commit is contained in:
Simon Detheridge 2019-12-16 10:42:31 +00:00
parent 601861ee58
commit 6225f2f236
14 changed files with 1464 additions and 1139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

190
services/filestore/app/js/AWSSDKPersistorManager.js
View file

@ -15,137 +15,183 @@
// latest aws-sdk and delete this module so that PersistorManager would load the
// same backend for both the 's3' and 'aws-sdk' options.
const logger = require("logger-sharelatex");
const aws = require("aws-sdk");
const _ = require("underscore");
const fs = require("fs");
const Errors = require("./Errors");
const logger = require('logger-sharelatex')
const aws = require('aws-sdk')
const _ = require('underscore')
const fs = require('fs')
const Errors = require('./Errors')
const s3 = new aws.S3();
const s3 = new aws.S3()
module.exports = {
sendFile(bucketName, key, fsPath, callback) {
logger.log({bucketName, key}, "send file data to s3");
const stream = fs.createReadStream(fsPath);
return s3.upload({Bucket: bucketName, Key: key, Body: stream}, function(err, data) {
logger.log({ bucketName, key }, 'send file data to s3')
const stream = fs.createReadStream(fsPath)
return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function(
err,
data
) {
if (err != null) {
logger.err({err, Bucket: bucketName, Key: key}, "error sending file data to s3");
logger.err(
{ err, Bucket: bucketName, Key: key },
'error sending file data to s3'
)
}
return callback(err);
});
return callback(err)
})
},
sendStream(bucketName, key, stream, callback) {
logger.log({bucketName, key}, "send file stream to s3");
return s3.upload({Bucket: bucketName, Key: key, Body: stream}, function(err, data) {
logger.log({ bucketName, key }, 'send file stream to s3')
return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function(
err,
data
) {
if (err != null) {
logger.err({err, Bucket: bucketName, Key: key}, "error sending file stream to s3");
logger.err(
{ err, Bucket: bucketName, Key: key },
'error sending file stream to s3'
)
}
return callback(err);
});
return callback(err)
})
},
getFileStream(bucketName, key, opts, callback) {
if (callback == null) { callback = function(err, res){}; }
logger.log({bucketName, key}, "get file stream from s3");
callback = _.once(callback);
if (callback == null) {
callback = function(err, res) {}
}
logger.log({ bucketName, key }, 'get file stream from s3')
callback = _.once(callback)
const params = {
Bucket: bucketName,
Key: key
};
if ((opts.start != null) && (opts.end != null)) {
params.Range = `bytes=${opts.start}-${opts.end}`;
}
const request = s3.getObject(params);
const stream = request.createReadStream();
stream.on('readable', () => callback(null, stream));
if (opts.start != null && opts.end != null) {
params.Range = `bytes=${opts.start}-${opts.end}`
}
const request = s3.getObject(params)
const stream = request.createReadStream()
stream.on('readable', () => callback(null, stream))
return stream.on('error', function(err) {
logger.err({err, bucketName, key}, "error getting file stream from s3");
logger.err({ err, bucketName, key }, 'error getting file stream from s3')
if (err.code === 'NoSuchKey') {
return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`));
return callback(
new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
)
}
return callback(err);
});
return callback(err)
})
},
copyFile(bucketName, sourceKey, destKey, callback) {
logger.log({bucketName, sourceKey, destKey}, "copying file in s3");
const source = bucketName + '/' + sourceKey;
return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) {
logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
const source = bucketName + '/' + sourceKey
return s3.copyObject(
{ Bucket: bucketName, Key: destKey, CopySource: source },
function(err) {
if (err != null) {
logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in s3");
logger.err(
{ err, bucketName, sourceKey, destKey },
'something went wrong copying file in s3'
)
}
return callback(err);
});
return callback(err)
}
)
},
deleteFile(bucketName, key, callback) {
logger.log({bucketName, key}, "delete file in s3");
logger.log({ bucketName, key }, 'delete file in s3')
return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) {
if (err != null) {
logger.err({err, bucketName, key}, "something went wrong deleting file in s3");
logger.err(
{ err, bucketName, key },
'something went wrong deleting file in s3'
)
}
return callback(err);
});
return callback(err)
})
},
deleteDirectory(bucketName, key, callback) {
logger.log({bucketName, key}, "delete directory in s3");
return s3.listObjects({Bucket: bucketName, Prefix: key}, function(err, data) {
logger.log({ bucketName, key }, 'delete directory in s3')
return s3.listObjects({ Bucket: bucketName, Prefix: key }, function(
err,
data
) {
if (err != null) {
logger.err({err, bucketName, key}, "something went wrong listing prefix in s3");
return callback(err);
logger.err(
{ err, bucketName, key },
'something went wrong listing prefix in s3'
)
return callback(err)
}
if (data.Contents.length === 0) {
logger.log({bucketName, key}, "the directory is empty");
return callback();
logger.log({ bucketName, key }, 'the directory is empty')
return callback()
}
const keys = _.map(data.Contents, entry => ({
Key: entry.Key
}));
return s3.deleteObjects({
}))
return s3.deleteObjects(
{
Bucket: bucketName,
Delete: {
Objects: keys,
Quiet: true
}
}
, function(err) {
},
function(err) {
if (err != null) {
logger.err({err, bucketName, key:keys}, "something went wrong deleting directory in s3");
logger.err(
{ err, bucketName, key: keys },
'something went wrong deleting directory in s3'
)
}
return callback(err);
});
});
return callback(err)
}
)
})
},
checkIfFileExists(bucketName, key, callback) {
logger.log({bucketName, key}, "check file existence in s3");
logger.log({ bucketName, key }, 'check file existence in s3')
return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
if (err != null) {
if (err.code === 'NotFound') { return (callback(null, false)); }
logger.err({err, bucketName, key}, "something went wrong checking head in s3");
return callback(err);
if (err.code === 'NotFound') {
return callback(null, false)
}
return callback(null, (data.ETag != null));
});
logger.err(
{ err, bucketName, key },
'something went wrong checking head in s3'
)
return callback(err)
}
return callback(null, data.ETag != null)
})
},
directorySize(bucketName, key, callback) {
logger.log({bucketName, key}, "get project size in s3");
return s3.listObjects({Bucket: bucketName, Prefix: key}, function(err, data) {
logger.log({ bucketName, key }, 'get project size in s3')
return s3.listObjects({ Bucket: bucketName, Prefix: key }, function(
err,
data
) {
if (err != null) {
logger.err({err, bucketName, key}, "something went wrong listing prefix in s3");
return callback(err);
logger.err(
{ err, bucketName, key },
'something went wrong listing prefix in s3'
)
return callback(err)
}
if (data.Contents.length === 0) {
logger.log({bucketName, key}, "the directory is empty");
return callback();
logger.log({ bucketName, key }, 'the directory is empty')
return callback()
}
let totalSize = 0
_.each(data.Contents, entry => (totalSize += entry.Size))
return callback(null, totalSize)
})
}
let totalSize = 0;
_.each(data.Contents, entry => totalSize += entry.Size);
return callback(null, totalSize);
});
}
};

44
services/filestore/app/js/BucketController.js
View file

@ -9,38 +9,40 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let BucketController;
const settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const FileHandler = require("./FileHandler");
const metrics = require("metrics-sharelatex");
const Errors = require('./Errors');
module.exports = (BucketController = {
let BucketController
const settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
const FileHandler = require('./FileHandler')
const metrics = require('metrics-sharelatex')
const Errors = require('./Errors')
module.exports = BucketController = {
getFile(req, res) {
const {bucket} = req.params;
const key = req.params[0];
const credentials = settings.filestore.s3BucketCreds != null ? settings.filestore.s3BucketCreds[bucket] : undefined;
const { bucket } = req.params
const key = req.params[0]
const credentials =
settings.filestore.s3BucketCreds != null
? settings.filestore.s3BucketCreds[bucket]
: undefined
const options = {
key,
bucket,
credentials
};
metrics.inc(`${bucket}.getFile`);
logger.log({key, bucket}, "receiving request to get file from bucket");
}
metrics.inc(`${bucket}.getFile`)
logger.log({ key, bucket }, 'receiving request to get file from bucket')
return FileHandler.getFile(bucket, key, options, function(err, fileStream) {
if (err != null) {
logger.err({err, key, bucket}, "problem getting file from bucket");
logger.err({ err, key, bucket }, 'problem getting file from bucket')
if (err instanceof Errors.NotFoundError) {
return res.send(404);
return res.send(404)
} else {
return res.send(500);
return res.send(500)
}
} else {
logger.log({key, bucket}, "sending bucket file to response");
return fileStream.pipe(res);
logger.log({ key, bucket }, 'sending bucket file to response')
return fileStream.pipe(res)
}
})
}
});
}
});

17
services/filestore/app/js/Errors.js
View file

@ -4,14 +4,13 @@
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
let Errors;
let Errors
var NotFoundError = function(message) {
const error = new Error(message);
error.name = "NotFoundError";
error.__proto__ = NotFoundError.prototype;
return error;
};
NotFoundError.prototype.__proto__ = Error.prototype;
const error = new Error(message)
error.name = 'NotFoundError'
error.__proto__ = NotFoundError.prototype
return error
}
NotFoundError.prototype.__proto__ = Error.prototype
module.exports = (Errors =
{NotFoundError});
module.exports = Errors = { NotFoundError }

221
services/filestore/app/js/FSPersistorManager.js
View file

@ -11,161 +11,196 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const logger = require("logger-sharelatex");
const fs = require("fs");
const path = require("path");
const LocalFileWriter = require("./LocalFileWriter");
const Errors = require('./Errors');
const rimraf = require("rimraf");
const _ = require("underscore");
const filterName = key => key.replace(/\//g, "_");
const logger = require('logger-sharelatex')
const fs = require('fs')
const path = require('path')
const LocalFileWriter = require('./LocalFileWriter')
const Errors = require('./Errors')
const rimraf = require('rimraf')
const _ = require('underscore')
const filterName = key => key.replace(/\//g, '_')
module.exports = {
sendFile(location, target, source, callback) {
if (callback == null) { callback = function(err){}; }
const filteredTarget = filterName(target);
logger.log({location, target:filteredTarget, source}, "sending file");
if (callback == null) {
callback = function(err) {}
}
const filteredTarget = filterName(target)
logger.log({ location, target: filteredTarget, source }, 'sending file')
const done = _.once(function(err) {
if (err != null) {
logger.err({err, location, target:filteredTarget, source}, "Error on put of file");
logger.err(
{ err, location, target: filteredTarget, source },
'Error on put of file'
)
}
return callback(err);
});
return callback(err)
})
// actually copy the file (instead of moving it) to maintain consistent behaviour
// between the different implementations
const sourceStream = fs.createReadStream(source);
sourceStream.on('error', done);
const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`);
targetStream.on('error', done);
targetStream.on('finish', () => done());
return sourceStream.pipe(targetStream);
const sourceStream = fs.createReadStream(source)
sourceStream.on('error', done)
const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`)
targetStream.on('error', done)
targetStream.on('finish', () => done())
return sourceStream.pipe(targetStream)
},
sendStream(location, target, sourceStream, callback) {
if (callback == null) { callback = function(err){}; }
logger.log({location, target}, "sending file stream");
sourceStream.on("error", err => logger.err({location, target, err:err("error on stream to send")}));
if (callback == null) {
callback = function(err) {}
}
logger.log({ location, target }, 'sending file stream')
sourceStream.on('error', err =>
logger.err({ location, target, err: err('error on stream to send') })
)
return LocalFileWriter.writeStream(sourceStream, null, (err, fsPath) => {
if (err != null) {
logger.err({location, target, fsPath, err}, "something went wrong writing stream to disk");
return callback(err);
logger.err(
{ location, target, fsPath, err },
'something went wrong writing stream to disk'
)
return callback(err)
}
return this.sendFile(location, target, fsPath, err => // delete the temporary file created above and return the original error
LocalFileWriter.deleteFile(fsPath, () => callback(err)));
});
return this.sendFile(location, target, fsPath, (
err // delete the temporary file created above and return the original error
) => LocalFileWriter.deleteFile(fsPath, () => callback(err)))
})
},
// opts may be {start: Number, end: Number}
getFileStream(location, name, opts, callback) {
if (callback == null) { callback = function(err, res){}; }
const filteredName = filterName(name);
logger.log({location, filteredName}, "getting file");
if (callback == null) {
callback = function(err, res) {}
}
const filteredName = filterName(name)
logger.log({ location, filteredName }, 'getting file')
return fs.open(`${location}/${filteredName}`, 'r', function(err, fd) {
if (err != null) {
logger.err({err, location, filteredName:name}, "Error reading from file");
logger.err(
{ err, location, filteredName: name },
'Error reading from file'
)
}
if (err.code === 'ENOENT') {
return callback(new Errors.NotFoundError(err.message), null);
return callback(new Errors.NotFoundError(err.message), null)
} else {
return callback(err, null);
return callback(err, null)
}
opts.fd = fd;
const sourceStream = fs.createReadStream(null, opts);
return callback(null, sourceStream);
});
opts.fd = fd
const sourceStream = fs.createReadStream(null, opts)
return callback(null, sourceStream)
})
},
getFileSize(location, filename, callback) {
const fullPath = path.join(location, filterName(filename));
const fullPath = path.join(location, filterName(filename))
return fs.stat(fullPath, function(err, stats) {
if (err != null) {
if (err.code === 'ENOENT') {
logger.log({location, filename}, "file not found");
callback(new Errors.NotFoundError(err.message));
logger.log({ location, filename }, 'file not found')
callback(new Errors.NotFoundError(err.message))
} else {
logger.err({err, location, filename}, "failed to stat file");
callback(err);
logger.err({ err, location, filename }, 'failed to stat file')
callback(err)
}
return;
return
}
return callback(null, stats.size);
});
return callback(null, stats.size)
})
},
copyFile(location, fromName, toName, callback) {
if (callback == null) { callback = function(err){}; }
const filteredFromName=filterName(fromName);
const filteredToName=filterName(toName);
logger.log({location, fromName:filteredFromName, toName:filteredToName}, "copying file");
const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`);
if (callback == null) {
callback = function(err) {}
}
const filteredFromName = filterName(fromName)
const filteredToName = filterName(toName)
logger.log(
{ location, fromName: filteredFromName, toName: filteredToName },
'copying file'
)
const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`)
sourceStream.on('error', function(err) {
logger.err({err, location, key:filteredFromName}, "Error reading from file");
return callback(err);
});
const targetStream = fs.createWriteStream(`${location}/${filteredToName}`);
logger.err(
{ err, location, key: filteredFromName },
'Error reading from file'
)
return callback(err)
})
const targetStream = fs.createWriteStream(`${location}/${filteredToName}`)
targetStream.on('error', function(err) {
logger.err({err, location, key:filteredToName}, "Error writing to file");
return callback(err);
});
targetStream.on('finish', () => callback(null));
return sourceStream.pipe(targetStream);
logger.err(
{ err, location, key: filteredToName },
'Error writing to file'
)
return callback(err)
})
targetStream.on('finish', () => callback(null))
return sourceStream.pipe(targetStream)
},
deleteFile(location, name, callback) {
const filteredName = filterName(name);
logger.log({location, filteredName}, "delete file");
const filteredName = filterName(name)
logger.log({ location, filteredName }, 'delete file')
return fs.unlink(`${location}/${filteredName}`, function(err) {
if (err != null) {
logger.err({err, location, filteredName}, "Error on delete.");
return callback(err);
logger.err({ err, location, filteredName }, 'Error on delete.')
return callback(err)
} else {
return callback();
return callback()
}
});
})
},
deleteDirectory(location, name, callback) {
if (callback == null) { callback = function(err){}; }
const filteredName = filterName(name.replace(/\/$/,''));
if (callback == null) {
callback = function(err) {}
}
const filteredName = filterName(name.replace(/\/$/, ''))
return rimraf(`${location}/${filteredName}`, function(err) {
if (err != null) {
logger.err({err, location, filteredName}, "Error on rimraf rmdir.");
return callback(err);
logger.err({ err, location, filteredName }, 'Error on rimraf rmdir.')
return callback(err)
} else {
return callback();
return callback()
}
});
})
},
checkIfFileExists(location, name, callback) {
if (callback == null) { callback = function(err,exists){}; }
const filteredName = filterName(name);
logger.log({location, filteredName}, "checking if file exists");
if (callback == null) {
callback = function(err, exists) {}
}
const filteredName = filterName(name)
logger.log({ location, filteredName }, 'checking if file exists')
return fs.exists(`${location}/${filteredName}`, function(exists) {
logger.log({location, filteredName, exists}, "checked if file exists");
return callback(null, exists);
});
logger.log({ location, filteredName, exists }, 'checked if file exists')
return callback(null, exists)
})
},
directorySize(location, name, callback) {
const filteredName = filterName(name.replace(/\/$/,''));
logger.log({location, filteredName}, "get project size in file system");
const filteredName = filterName(name.replace(/\/$/, ''))
logger.log({ location, filteredName }, 'get project size in file system')
return fs.readdir(`${location}/${filteredName}`, function(err, files) {
if (err != null) {
logger.err({err, location, filteredName}, "something went wrong listing prefix in aws");
return callback(err);
logger.err(
{ err, location, filteredName },
'something went wrong listing prefix in aws'
)
return callback(err)
}
let totalSize = 0;
let totalSize = 0
_.each(files, function(entry) {
const fd = fs.openSync(`${location}/${filteredName}/${entry}`, 'r');
const fileStats = fs.fstatSync(fd);
totalSize += fileStats.size;
return fs.closeSync(fd);
});
logger.log({totalSize}, "total size", {files});
return callback(null, totalSize);
});
const fd = fs.openSync(`${location}/${filteredName}/${entry}`, 'r')
const fileStats = fs.fstatSync(fd)
totalSize += fileStats.size
return fs.closeSync(fd)
})
logger.log({ totalSize }, 'total size', { files })
return callback(null, totalSize)
})
}
}
};

178
services/filestore/app/js/FileController.js
View file

@ -10,142 +10,164 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let FileController;
const PersistorManager = require("./PersistorManager");
const settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const FileHandler = require("./FileHandler");
const metrics = require("metrics-sharelatex");
const parseRange = require('range-parser');
const Errors = require('./Errors');
let FileController
const PersistorManager = require('./PersistorManager')
const settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
const FileHandler = require('./FileHandler')
const metrics = require('metrics-sharelatex')
const parseRange = require('range-parser')
const Errors = require('./Errors')
const oneDayInSeconds = 60 * 60 * 24;
const maxSizeInBytes = 1024 * 1024 * 1024; // 1GB
module.exports = (FileController = {
const oneDayInSeconds = 60 * 60 * 24
const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB
module.exports = FileController = {
getFile(req, res) {
const {key, bucket} = req;
const {format, style} = req.query;
const { key, bucket } = req
const { format, style } = req.query
const options = {
key,
bucket,
format,
style,
};
metrics.inc("getFile");
logger.log({key, bucket, format, style}, "receiving request to get file");
style
}
metrics.inc('getFile')
logger.log({ key, bucket, format, style }, 'receiving request to get file')
if (req.headers.range != null) {
const range = FileController._get_range(req.headers.range);
options.start = range.start;
options.end = range.end;
logger.log({start: range.start, end: range.end}, "getting range of bytes from file");
const range = FileController._get_range(req.headers.range)
options.start = range.start
options.end = range.end
logger.log(
{ start: range.start, end: range.end },
'getting range of bytes from file'
)
}
return FileHandler.getFile(bucket, key, options, function(err, fileStream) {
if (err != null) {
if (err instanceof Errors.NotFoundError) {
return res.send(404);
return res.send(404)
} else {
logger.err({err, key, bucket, format, style}, "problem getting file");
return res.send(500);
logger.err(
{ err, key, bucket, format, style },
'problem getting file'
)
return res.send(500)
}
} else if (req.query.cacheWarm) {
logger.log({key, bucket, format, style}, "request is only for cache warm so not sending stream");
return res.send(200);
logger.log(
{ key, bucket, format, style },
'request is only for cache warm so not sending stream'
)
return res.send(200)
} else {
logger.log({key, bucket, format, style}, "sending file to response");
return fileStream.pipe(res);
logger.log({ key, bucket, format, style }, 'sending file to response')
return fileStream.pipe(res)
}
});
})
},
getFileHead(req, res) {
const {key, bucket} = req;
metrics.inc("getFileSize");
logger.log({ key, bucket }, "receiving request to get file metadata");
const { key, bucket } = req
metrics.inc('getFileSize')
logger.log({ key, bucket }, 'receiving request to get file metadata')
return FileHandler.getFileSize(bucket, key, function(err, fileSize) {
if (err != null) {
if (err instanceof Errors.NotFoundError) {
res.status(404).end();
res.status(404).end()
} else {
res.status(500).end();
res.status(500).end()
}
return;
return
}
res.set("Content-Length", fileSize);
return res.status(200).end();
});
res.set('Content-Length', fileSize)
return res.status(200).end()
})
},
insertFile(req, res) {
metrics.inc("insertFile");
const {key, bucket} = req;
logger.log({key, bucket}, "receiving request to insert file");
metrics.inc('insertFile')
const { key, bucket } = req
logger.log({ key, bucket }, 'receiving request to insert file')
return FileHandler.insertFile(bucket, key, req, function(err) {
if (err != null) {
logger.log({err, key, bucket}, "error inserting file");
return res.send(500);
logger.log({ err, key, bucket }, 'error inserting file')
return res.send(500)
} else {
return res.send(200);
return res.send(200)
}
});
})
},
copyFile(req, res) {
metrics.inc("copyFile");
const {key, bucket} = req;
const oldProject_id = req.body.source.project_id;
const oldFile_id = req.body.source.file_id;
logger.log({key, bucket, oldProject_id, oldFile_id}, "receiving request to copy file");
return PersistorManager.copyFile(bucket, `${oldProject_id}/${oldFile_id}`, key, function(err){
metrics.inc('copyFile')
const { key, bucket } = req
const oldProject_id = req.body.source.project_id
const oldFile_id = req.body.source.file_id
logger.log(
{ key, bucket, oldProject_id, oldFile_id },
'receiving request to copy file'
)
return PersistorManager.copyFile(
bucket,
`${oldProject_id}/${oldFile_id}`,
key,
function(err) {
if (err != null) {
if (err instanceof Errors.NotFoundError) {
return res.send(404);
return res.send(404)
} else {
logger.log({err, oldProject_id, oldFile_id}, "something went wrong copying file");
return res.send(500);
logger.log(
{ err, oldProject_id, oldFile_id },
'something went wrong copying file'
)
return res.send(500)
}
} else {
return res.send(200);
return res.send(200)
}
});
}
)
},
deleteFile(req, res) {
metrics.inc("deleteFile");
const {key, bucket} = req;
logger.log({key, bucket}, "receiving request to delete file");
metrics.inc('deleteFile')
const { key, bucket } = req
logger.log({ key, bucket }, 'receiving request to delete file')
return FileHandler.deleteFile(bucket, key, function(err) {
if (err != null) {
logger.log({err, key, bucket}, "something went wrong deleting file");
return res.send(500);
logger.log({ err, key, bucket }, 'something went wrong deleting file')
return res.send(500)
} else {
return res.send(204);
return res.send(204)
}
});
})
},
_get_range(header) {
const parsed = parseRange(maxSizeInBytes, header);
if ((parsed === -1) || (parsed === -2) || (parsed.type !== 'bytes')) {
return null;
const parsed = parseRange(maxSizeInBytes, header)
if (parsed === -1 || parsed === -2 || parsed.type !== 'bytes') {
return null
} else {
const range = parsed[0];
return {start: range.start, end: range.end};
const range = parsed[0]
return { start: range.start, end: range.end }
}
},
directorySize(req, res) {
metrics.inc("projectSize");
const {project_id, bucket} = req;
logger.log({project_id, bucket}, "receiving request to project size");
return FileHandler.getDirectorySize(bucket, project_id, function(err, size){
metrics.inc('projectSize')
const { project_id, bucket } = req
logger.log({ project_id, bucket }, 'receiving request to project size')
return FileHandler.getDirectorySize(bucket, project_id, function(
err,
size
) {
if (err != null) {
logger.log({err, project_id, bucket}, "error inserting file");
return res.send(500);
logger.log({ err, project_id, bucket }, 'error inserting file')
return res.send(500)
} else {
return res.json({'total bytes' : size});
return res.json({ 'total bytes': size })
}
})
}
});
}
});

140
services/filestore/app/js/FileConverter.js
View file

@ -9,77 +9,125 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const _ = require("underscore");
const metrics = require("metrics-sharelatex");
const logger = require("logger-sharelatex");
const safe_exec = require("./SafeExec");
const approvedFormats = ["png"];
const Settings = require("settings-sharelatex");
const _ = require('underscore')
const metrics = require('metrics-sharelatex')
const logger = require('logger-sharelatex')
const safe_exec = require('./SafeExec')
const approvedFormats = ['png']
const Settings = require('settings-sharelatex')
const fourtySeconds = 40 * 1000;
const fourtySeconds = 40 * 1000
const childProcessOpts = {
killSignal: "SIGTERM",
killSignal: 'SIGTERM',
timeout: fourtySeconds
};
}
module.exports = {
convert(sourcePath, requestedFormat, callback) {
logger.log({sourcePath, requestedFormat}, "converting file format");
const timer = new metrics.Timer("imageConvert");
const destPath = `${sourcePath}.${requestedFormat}`;
sourcePath = `${sourcePath}[0]`;
logger.log({ sourcePath, requestedFormat }, 'converting file format')
const timer = new metrics.Timer('imageConvert')
const destPath = `${sourcePath}.${requestedFormat}`
sourcePath = `${sourcePath}[0]`
if (!_.include(approvedFormats, requestedFormat)) {
const err = new Error("invalid format requested");
return callback(err);
const err = new Error('invalid format requested')
return callback(err)
}
const width = "600x";
let command = ["convert", "-define", `pdf:fit-page=${width}`, "-flatten", "-density", "300", sourcePath, destPath];
command = Settings.commands.convertCommandPrefix.concat(command);
const width = '600x'
let command = [
'convert',
'-define',
`pdf:fit-page=${width}`,
'-flatten',
'-density',
'300',
sourcePath,
destPath
]
command = Settings.commands.convertCommandPrefix.concat(command)
return safe_exec(command, childProcessOpts, function(err, stdout, stderr) {
timer.done();
timer.done()
if (err != null) {
logger.err({err, stderr, sourcePath, requestedFormat, destPath}, "something went wrong converting file");
logger.err(
{ err, stderr, sourcePath, requestedFormat, destPath },
'something went wrong converting file'
)
} else {
logger.log({sourcePath, requestedFormat, destPath}, "finished converting file");
logger.log(
{ sourcePath, requestedFormat, destPath },
'finished converting file'
)
}
return callback(err, destPath);
});
return callback(err, destPath)
})
},
thumbnail(sourcePath, callback) {
const destPath = `${sourcePath}.png`;
sourcePath = `${sourcePath}[0]`;
const width = "260x";
let command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", `pdf:fit-page=${width}`, sourcePath, "-resize", width, destPath];
logger.log({sourcePath, destPath, command}, "thumbnail convert file");
command = Settings.commands.convertCommandPrefix.concat(command);
const destPath = `${sourcePath}.png`
sourcePath = `${sourcePath}[0]`
const width = '260x'
let command = [
'convert',
'-flatten',
'-background',
'white',
'-density',
'300',
'-define',
`pdf:fit-page=${width}`,
sourcePath,
'-resize',
width,
destPath
]
logger.log({ sourcePath, destPath, command }, 'thumbnail convert file')
command = Settings.commands.convertCommandPrefix.concat(command)
return safe_exec(command, childProcessOpts, function(err, stdout, stderr) {
if (err != null) {
logger.err({err, stderr, sourcePath}, "something went wrong converting file to thumbnail");
logger.err(
{ err, stderr, sourcePath },
'something went wrong converting file to thumbnail'
)
} else {
logger.log({sourcePath, destPath}, "finished thumbnailing file");
logger.log({ sourcePath, destPath }, 'finished thumbnailing file')
}
return callback(err, destPath);
});
return callback(err, destPath)
})
},
preview(sourcePath, callback) {
logger.log({sourcePath}, "preview convert file");
const destPath = `${sourcePath}.png`;
sourcePath = `${sourcePath}[0]`;
const width = "548x";
let command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", `pdf:fit-page=${width}`, sourcePath, "-resize", width, destPath];
command = Settings.commands.convertCommandPrefix.concat(command);
logger.log({ sourcePath }, 'preview convert file')
const destPath = `${sourcePath}.png`
sourcePath = `${sourcePath}[0]`
const width = '548x'
let command = [
'convert',
'-flatten',
'-background',
'white',
'-density',
'300',
'-define',
`pdf:fit-page=${width}`,
sourcePath,
'-resize',
width,
destPath
]
command = Settings.commands.convertCommandPrefix.concat(command)
return safe_exec(command, childProcessOpts, function(err, stdout, stderr) {
if (err != null) {
logger.err({err, stderr, sourcePath, destPath}, "something went wrong converting file to preview");
logger.err(
{ err, stderr, sourcePath, destPath },
'something went wrong converting file to preview'
)
} else {
logger.log({sourcePath, destPath}, "finished converting file to preview");
logger.log(
{ sourcePath, destPath },
'finished converting file to preview'
)
}
return callback(err, destPath)
})
}
return callback(err, destPath);
});
}
};

233
services/filestore/app/js/FileHandler.js
View file

@ -11,91 +11,127 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let FileHandler;
const settings = require("settings-sharelatex");
const PersistorManager = require("./PersistorManager");
const LocalFileWriter = require("./LocalFileWriter");
const logger = require("logger-sharelatex");
const FileConverter = require("./FileConverter");
const KeyBuilder = require("./KeyBuilder");
const async = require("async");
const ImageOptimiser = require("./ImageOptimiser");
const Errors = require('./Errors');
module.exports = (FileHandler = {
let FileHandler
const settings = require('settings-sharelatex')
const PersistorManager = require('./PersistorManager')
const LocalFileWriter = require('./LocalFileWriter')
const logger = require('logger-sharelatex')
const FileConverter = require('./FileConverter')
const KeyBuilder = require('./KeyBuilder')
const async = require('async')
const ImageOptimiser = require('./ImageOptimiser')
const Errors = require('./Errors')
module.exports = FileHandler = {
insertFile(bucket, key, stream, callback) {
const convertedKey = KeyBuilder.getConvertedFolderKey(key);
return PersistorManager.deleteDirectory(bucket, convertedKey, function(error) {
if (error != null) { return callback(error); }
return PersistorManager.sendStream(bucket, key, stream, callback);
});
const convertedKey = KeyBuilder.getConvertedFolderKey(key)
return PersistorManager.deleteDirectory(bucket, convertedKey, function(
error
) {
if (error != null) {
return callback(error)
}
return PersistorManager.sendStream(bucket, key, stream, callback)
})
},
deleteFile(bucket, key, callback) {
const convertedKey = KeyBuilder.getConvertedFolderKey(key);
return async.parallel([
const convertedKey = KeyBuilder.getConvertedFolderKey(key)
return async.parallel(
[
done => PersistorManager.deleteFile(bucket, key, done),
done => PersistorManager.deleteDirectory(bucket, convertedKey, done)
], callback);
],
callback
)
},
getFile(bucket, key, opts, callback) {
// In this call, opts can contain credentials
if (opts == null) { opts = {}; }
logger.log({bucket, key, opts:this._scrubSecrets(opts)}, "getting file");
if ((opts.format == null) && (opts.style == null)) {
return this._getStandardFile(bucket, key, opts, callback);
if (opts == null) {
opts = {}
}
logger.log({ bucket, key, opts: this._scrubSecrets(opts) }, 'getting file')
if (opts.format == null && opts.style == null) {
return this._getStandardFile(bucket, key, opts, callback)
} else {
return this._getConvertedFile(bucket, key, opts, callback);
return this._getConvertedFile(bucket, key, opts, callback)
}
},
getFileSize(bucket, key, callback) {
return PersistorManager.getFileSize(bucket, key, callback);
return PersistorManager.getFileSize(bucket, key, callback)
},
_getStandardFile(bucket, key, opts, callback) {
return PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream){
if ((err != null) && !(err instanceof Errors.NotFoundError)) {
logger.err({bucket, key, opts:FileHandler._scrubSecrets(opts)}, "error getting fileStream");
return PersistorManager.getFileStream(bucket, key, opts, function(
err,
fileStream
) {
if (err != null && !(err instanceof Errors.NotFoundError)) {
logger.err(
{ bucket, key, opts: FileHandler._scrubSecrets(opts) },
'error getting fileStream'
)
}
return callback(err, fileStream);
});
return callback(err, fileStream)
})
},
_getConvertedFile(bucket, key, opts, callback) {
const convertedKey = KeyBuilder.addCachingToKey(key, opts);
return PersistorManager.checkIfFileExists(bucket, convertedKey, (err, exists)=> {
const convertedKey = KeyBuilder.addCachingToKey(key, opts)
return PersistorManager.checkIfFileExists(
bucket,
convertedKey,
(err, exists) => {
if (err != null) {
return callback(err);
return callback(err)
}
if (exists) {
return PersistorManager.getFileStream(bucket, convertedKey, opts, callback);
return PersistorManager.getFileStream(
bucket,
convertedKey,
opts,
callback
)
} else {
return this._getConvertedFileAndCache(bucket, key, convertedKey, opts, callback);
return this._getConvertedFileAndCache(
bucket,
key,
convertedKey,
opts,
callback
)
}
});
}
)
},
_getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) {
let convertedFsPath = "";
const originalFsPath = "";
return async.series([
let convertedFsPath = ''
const originalFsPath = ''
return async.series(
[
cb => {
return this._convertFile(bucket, key, opts, function(err, fileSystemPath, originalFsPath) {
convertedFsPath = fileSystemPath;
originalFsPath = originalFsPath;
return cb(err);
});
return this._convertFile(bucket, key, opts, function(
err,
fileSystemPath,
originalFsPath
) {
convertedFsPath = fileSystemPath
originalFsPath = originalFsPath
return cb(err)
})
},
cb => ImageOptimiser.compressPng(convertedFsPath, cb),
cb => PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb)
], function(err){
cb =>
PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb)
],
function(err) {
if (err != null) {
LocalFileWriter.deleteFile(convertedFsPath, function() {});
LocalFileWriter.deleteFile(originalFsPath, function() {});
return callback(err);
LocalFileWriter.deleteFile(convertedFsPath, function() {})
LocalFileWriter.deleteFile(originalFsPath, function() {})
return callback(err)
}
// Send back the converted file from the local copy to avoid problems
// with the file not being present in S3 yet. As described in the
@ -109,68 +145,91 @@ module.exports = (FileHandler = {
// (to find if the object exists) before creating the object, Amazon
// S3 provides eventual consistency for read-after-write.""
// https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
return LocalFileWriter.getStream(convertedFsPath, function(err, readStream) {
if (err != null) { return callback(err); }
return LocalFileWriter.getStream(convertedFsPath, function(
err,
readStream
) {
if (err != null) {
return callback(err)
}
readStream.on('end', function() {
logger.log({convertedFsPath}, "deleting temporary file");
return LocalFileWriter.deleteFile(convertedFsPath, function() {});
});
return callback(null, readStream);
});
});
logger.log({ convertedFsPath }, 'deleting temporary file')
return LocalFileWriter.deleteFile(convertedFsPath, function() {})
})
return callback(null, readStream)
})
}
)
},
_convertFile(bucket, originalKey, opts, callback) {
return this._writeS3FileToDisk(bucket, originalKey, opts, function(err, originalFsPath){
return this._writeS3FileToDisk(bucket, originalKey, opts, function(
err,
originalFsPath
) {
if (err != null) {
return callback(err);
return callback(err)
}
const done = function(err, destPath) {
if (err != null) {
logger.err({err, bucket, originalKey, opts:FileHandler._scrubSecrets(opts)}, "error converting file");
return callback(err);
logger.err(
{ err, bucket, originalKey, opts: FileHandler._scrubSecrets(opts) },
'error converting file'
)
return callback(err)
}
LocalFileWriter.deleteFile(originalFsPath, function() {})
return callback(err, destPath, originalFsPath)
}
LocalFileWriter.deleteFile(originalFsPath, function() {});
return callback(err, destPath, originalFsPath);
};
logger.log({opts}, "converting file depending on opts");
logger.log({ opts }, 'converting file depending on opts')
if (opts.format != null) {
return FileConverter.convert(originalFsPath, opts.format, done);
} else if (opts.style === "thumbnail") {
return FileConverter.thumbnail(originalFsPath, done);
} else if (opts.style === "preview") {
return FileConverter.preview(originalFsPath, done);
return FileConverter.convert(originalFsPath, opts.format, done)
} else if (opts.style === 'thumbnail') {
return FileConverter.thumbnail(originalFsPath, done)
} else if (opts.style === 'preview') {
return FileConverter.preview(originalFsPath, done)
} else {
return callback(new Error(`should have specified opts to convert file with ${JSON.stringify(opts)}`));
return callback(
new Error(
`should have specified opts to convert file with ${JSON.stringify(
opts
)}`
)
)
}
});
})
},
_writeS3FileToDisk(bucket, key, opts, callback) {
return PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream){
return PersistorManager.getFileStream(bucket, key, opts, function(
err,
fileStream
) {
if (err != null) {
return callback(err);
return callback(err)
}
return LocalFileWriter.writeStream(fileStream, key, callback);
});
return LocalFileWriter.writeStream(fileStream, key, callback)
})
},
getDirectorySize(bucket, project_id, callback) {
logger.log({bucket, project_id}, "getting project size");
return PersistorManager.directorySize(bucket, project_id, function(err, size){
logger.log({ bucket, project_id }, 'getting project size')
return PersistorManager.directorySize(bucket, project_id, function(
err,
size
) {
if (err != null) {
logger.err({bucket, project_id}, "error getting size");
logger.err({ bucket, project_id }, 'error getting size')
}
return callback(err, size);
});
return callback(err, size)
})
},
_scrubSecrets(opts) {
const safe = Object.assign({}, opts);
delete safe.credentials;
return safe;
const safe = Object.assign({}, opts)
delete safe.credentials
return safe
}
}
});

90
services/filestore/app/js/HealthCheckController.js
View file

@ -6,71 +6,75 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const fs = require("fs-extra");
const path = require("path");
const async = require("async");
const fileConverter = require("./FileConverter");
const keyBuilder = require("./KeyBuilder");
const fileController = require("./FileController");
const logger = require('logger-sharelatex');
const settings = require("settings-sharelatex");
const streamBuffers = require("stream-buffers");
const _ = require('underscore');
const fs = require('fs-extra')
const path = require('path')
const async = require('async')
const fileConverter = require('./FileConverter')
const keyBuilder = require('./KeyBuilder')
const fileController = require('./FileController')
const logger = require('logger-sharelatex')
const settings = require('settings-sharelatex')
const streamBuffers = require('stream-buffers')
const _ = require('underscore')
const checkCanStoreFiles = function(callback) {
callback = _.once(callback);
const req = {params:{}, query:{}, headers:{}};
req.params.project_id = settings.health_check.project_id;
req.params.file_id = settings.health_check.file_id;
const myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer({initialSize: 100});
callback = _.once(callback)
const req = { params: {}, query: {}, headers: {} }
req.params.project_id = settings.health_check.project_id
req.params.file_id = settings.health_check.file_id
const myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer({
initialSize: 100
})
const res = {
send(code) {
if (code !== 200) {
return callback(new Error(`non-200 code from getFile: ${code}`));
return callback(new Error(`non-200 code from getFile: ${code}`))
}
}
};
myWritableStreamBuffer.send = res.send;
}
myWritableStreamBuffer.send = res.send
return keyBuilder.userFileKey(req, res, function() {
fileController.getFile(req, myWritableStreamBuffer);
return myWritableStreamBuffer.on("close", function() {
fileController.getFile(req, myWritableStreamBuffer)
return myWritableStreamBuffer.on('close', function() {
if (myWritableStreamBuffer.size() > 0) {
return callback();
return callback()
} else {
const err = "no data in write stream buffer for health check";
logger.err({err,}, "error performing health check");
return callback(err);
const err = 'no data in write stream buffer for health check'
logger.err({ err }, 'error performing health check')
return callback(err)
}
})
})
}
});
});
};
const checkFileConvert = function(callback) {
if (!settings.enableConversions) {
return callback();
return callback()
}
const imgPath = path.join(settings.path.uploadFolder, "/tiny.pdf");
return async.waterfall([
cb => fs.copy("./tiny.pdf", imgPath, cb),
const imgPath = path.join(settings.path.uploadFolder, '/tiny.pdf')
return async.waterfall(
[
cb => fs.copy('./tiny.pdf', imgPath, cb),
cb => fileConverter.thumbnail(imgPath, cb),
(resultPath, cb) => fs.unlink(resultPath, cb),
cb => fs.unlink(imgPath, cb)
], callback);
};
],
callback
)
}
module.exports = {
check(req, res) {
logger.log({}, "performing health check");
return async.parallel([checkFileConvert, checkCanStoreFiles], function(err){
logger.log({}, 'performing health check')
return async.parallel([checkFileConvert, checkCanStoreFiles], function(
err
) {
if (err != null) {
logger.err({err}, "Health check: error running");
return res.send(500);
logger.err({ err }, 'Health check: error running')
return res.send(500)
} else {
return res.send(200);
return res.send(200)
}
})
}
});
}
};

42
services/filestore/app/js/ImageOptimiser.js
View file

@ -9,36 +9,36 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const {
exec
} = require('child_process');
const logger = require("logger-sharelatex");
const Settings = require("settings-sharelatex");
const { exec } = require('child_process')
const logger = require('logger-sharelatex')
const Settings = require('settings-sharelatex')
module.exports = {
compressPng(localPath, callback) {
const startTime = new Date();
logger.log({localPath}, "optimising png path");
const args = `optipng ${localPath}`;
const startTime = new Date()
logger.log({ localPath }, 'optimising png path')
const args = `optipng ${localPath}`
const opts = {
timeout: 30 * 1000,
killSignal: "SIGKILL"
};
killSignal: 'SIGKILL'
}
if (!Settings.enableConversions) {
const error = new Error("Image conversions are disabled");
return callback(error);
const error = new Error('Image conversions are disabled')
return callback(error)
}
return exec(args, opts, function(err, stdout, stderr) {
if ((err != null) && (err.signal === 'SIGKILL')) {
logger.warn({err, stderr, localPath}, "optimiser timeout reached");
err = null;
if (err != null && err.signal === 'SIGKILL') {
logger.warn({ err, stderr, localPath }, 'optimiser timeout reached')
err = null
} else if (err != null) {
logger.err({err, stderr, localPath}, "something went wrong converting compressPng");
logger.err(
{ err, stderr, localPath },
'something went wrong converting compressPng'
)
} else {
logger.log({localPath}, "finished compressPng file");
logger.log({ localPath }, 'finished compressPng file')
}
return callback(err)
})
}
return callback(err);
});
}
};

68
services/filestore/app/js/KeyBuilder.js
View file

@ -11,65 +11,61 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const settings = require("settings-sharelatex");
const settings = require('settings-sharelatex')
module.exports = {
getConvertedFolderKey(key) {
return key = `${key}-converted-cache/`;
return (key = `${key}-converted-cache/`)
},
addCachingToKey(key, opts) {
key = this.getConvertedFolderKey(key);
if ((opts.format != null) && (opts.style == null)) {
key = `${key}format-${opts.format}`;
key = this.getConvertedFolderKey(key)
if (opts.format != null && opts.style == null) {
key = `${key}format-${opts.format}`
}
if ((opts.style != null) && (opts.format == null)) {
key = `${key}style-${opts.style}`;
if (opts.style != null && opts.format == null) {
key = `${key}style-${opts.style}`
}
if ((opts.style != null) && (opts.format != null)) {
key = `${key}format-${opts.format}-style-${opts.style}`;
if (opts.style != null && opts.format != null) {
key = `${key}format-${opts.format}-style-${opts.style}`
}
return key;
return key
},
userFileKey(req, res, next) {
const {project_id, file_id} = req.params;
req.key = `${project_id}/${file_id}`;
req.bucket = settings.filestore.stores.user_files;
return next();
const { project_id, file_id } = req.params
req.key = `${project_id}/${file_id}`
req.bucket = settings.filestore.stores.user_files
return next()
},
publicFileKey(req, res, next) {
const {project_id, public_file_id} = req.params;
if ((settings.filestore.stores.public_files == null)) {
return res.status(501).send("public files not available");
const { project_id, public_file_id } = req.params
if (settings.filestore.stores.public_files == null) {
return res.status(501).send('public files not available')
} else {
req.key = `${project_id}/${public_file_id}`;
req.bucket = settings.filestore.stores.public_files;
return next();
req.key = `${project_id}/${public_file_id}`
req.bucket = settings.filestore.stores.public_files
return next()
}
},
templateFileKey(req, res, next) {
const {template_id, format, version, sub_type} = req.params;
req.key = `${template_id}/v/${version}/${format}`;
const { template_id, format, version, sub_type } = req.params
req.key = `${template_id}/v/${version}/${format}`
if (sub_type != null) {
req.key = `${req.key}/${sub_type}`;
req.key = `${req.key}/${sub_type}`
}
req.bucket = settings.filestore.stores.template_files;
req.version = version;
const opts = req.query;
return next();
req.bucket = settings.filestore.stores.template_files
req.version = version
const opts = req.query
return next()
},
publicProjectKey(req, res, next) {
const {project_id} = req.params;
req.project_id = project_id;
req.bucket = settings.filestore.stores.user_files;
return next();
const { project_id } = req.params
req.project_id = project_id
req.bucket = settings.filestore.stores.user_files
return next()
}
}
};

118
services/filestore/app/js/LocalFileWriter.js
View file

@ -9,73 +9,83 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const fs = require("fs");
const uuid = require('node-uuid');
const path = require("path");
const _ = require("underscore");
const logger = require("logger-sharelatex");
const metrics = require("metrics-sharelatex");
const Settings = require("settings-sharelatex");
const Errors = require("./Errors");
const fs = require('fs')
const uuid = require('node-uuid')
const path = require('path')
const _ = require('underscore')
const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex')
const Settings = require('settings-sharelatex')
const Errors = require('./Errors')
module.exports = {
writeStream(stream, key, callback) {
const timer = new metrics.Timer("writingFile");
callback = _.once(callback);
const fsPath = this._getPath(key);
logger.log({fsPath}, "writing file locally");
const writeStream = fs.createWriteStream(fsPath);
writeStream.on("finish", function() {
timer.done();
logger.log({fsPath}, "finished writing file locally");
return callback(null, fsPath);
});
writeStream.on("error", function(err){
logger.err({err, fsPath}, "problem writing file locally, with write stream");
return callback(err);
});
stream.on("error", function(err){
logger.log({err, fsPath}, "problem writing file locally, with read stream");
return callback(err);
});
return stream.pipe(writeStream);
const timer = new metrics.Timer('writingFile')
callback = _.once(callback)
const fsPath = this._getPath(key)
logger.log({ fsPath }, 'writing file locally')
const writeStream = fs.createWriteStream(fsPath)
writeStream.on('finish', function() {
timer.done()
logger.log({ fsPath }, 'finished writing file locally')
return callback(null, fsPath)
})
writeStream.on('error', function(err) {
logger.err(
{ err, fsPath },
'problem writing file locally, with write stream'
)
return callback(err)
})
stream.on('error', function(err) {
logger.log(
{ err, fsPath },
'problem writing file locally, with read stream'
)
return callback(err)
})
return stream.pipe(writeStream)
},
getStream(fsPath, _callback) {
if (_callback == null) { _callback = function(err, res){}; }
const callback = _.once(_callback);
const timer = new metrics.Timer("readingFile");
logger.log({fsPath}, "reading file locally");
const readStream = fs.createReadStream(fsPath);
readStream.on("end", function() {
timer.done();
return logger.log({fsPath}, "finished reading file locally");
});
readStream.on("error", function(err){
logger.err({err, fsPath}, "problem reading file locally, with read stream");
if (err.code === 'ENOENT') {
return callback(new Errors.NotFoundError(err.message), null);
} else {
return callback(err);
if (_callback == null) {
_callback = function(err, res) {}
}
});
return callback(null, readStream);
const callback = _.once(_callback)
const timer = new metrics.Timer('readingFile')
logger.log({ fsPath }, 'reading file locally')
const readStream = fs.createReadStream(fsPath)
readStream.on('end', function() {
timer.done()
return logger.log({ fsPath }, 'finished reading file locally')
})
readStream.on('error', function(err) {
logger.err(
{ err, fsPath },
'problem reading file locally, with read stream'
)
if (err.code === 'ENOENT') {
return callback(new Errors.NotFoundError(err.message), null)
} else {
return callback(err)
}
})
return callback(null, readStream)
},
deleteFile(fsPath, callback) {
if ((fsPath == null) || (fsPath === "")) {
return callback();
if (fsPath == null || fsPath === '') {
return callback()
}
logger.log({fsPath}, "removing local temp file");
return fs.unlink(fsPath, callback);
logger.log({ fsPath }, 'removing local temp file')
return fs.unlink(fsPath, callback)
},
_getPath(key) {
if ((key == null)) {
key = uuid.v1();
if (key == null) {
key = uuid.v1()
}
key = key.replace(/\//g, '-')
return path.join(Settings.path.uploadFolder, key)
}
key = key.replace(/\//g,"-");
return path.join(Settings.path.uploadFolder, key);
}
};

50
services/filestore/app/js/PersistorManager.js
View file

@ -7,24 +7,46 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
// assume s3 if none specified
__guard__(settings != null ? settings.filestore : undefined, x => x.backend || (settings.filestore.backend = "s3"));
__guard__(
settings != null ? settings.filestore : undefined,
x => x.backend || (settings.filestore.backend = 's3')
)
logger.log({backend:__guard__(settings != null ? settings.filestore : undefined, x1 => x1.backend)}, "Loading backend");
module.exports = (() => { switch (__guard__(settings != null ? settings.filestore : undefined, x2 => x2.backend)) {
case "aws-sdk":
return require("./AWSSDKPersistorManager");
case "s3":
return require("./S3PersistorManager");
case "fs":
return require("./FSPersistorManager");
logger.log(
{
backend: __guard__(
settings != null ? settings.filestore : undefined,
x1 => x1.backend
)
},
'Loading backend'
)
module.exports = (() => {
switch (
__guard__(
settings != null ? settings.filestore : undefined,
x2 => x2.backend
)
) {
case 'aws-sdk':
return require('./AWSSDKPersistorManager')
case 's3':
return require('./S3PersistorManager')
case 'fs':
return require('./FSPersistorManager')
default:
throw new Error( `Unknown filestore backend: ${settings.filestore.backend}` );
} })();
throw new Error(
`Unknown filestore backend: ${settings.filestore.backend}`
)
}
})()
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

374
services/filestore/app/js/S3PersistorManager.js
View file

@ -19,31 +19,31 @@
// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
// details. The knox library is unmaintained and has bugs.
const http = require('http');
http.globalAgent.maxSockets = 300;
const https = require('https');
https.globalAgent.maxSockets = 300;
const settings = require("settings-sharelatex");
const request = require("request");
const logger = require("logger-sharelatex");
const metrics = require("metrics-sharelatex");
const fs = require("fs");
const knox = require("knox");
const path = require("path");
const LocalFileWriter = require("./LocalFileWriter");
const Errors = require("./Errors");
const _ = require("underscore");
const awsS3 = require("aws-sdk/clients/s3");
const URL = require('url');
const http = require('http')
http.globalAgent.maxSockets = 300
const https = require('https')
https.globalAgent.maxSockets = 300
const settings = require('settings-sharelatex')
const request = require('request')
const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex')
const fs = require('fs')
const knox = require('knox')
const path = require('path')
const LocalFileWriter = require('./LocalFileWriter')
const Errors = require('./Errors')
const _ = require('underscore')
const awsS3 = require('aws-sdk/clients/s3')
const URL = require('url')
const thirtySeconds = 30 * 1000;
const thirtySeconds = 30 * 1000
const buildDefaultOptions = function(bucketName, method, key) {
let endpoint;
let endpoint
if (settings.filestore.s3.endpoint) {
endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`;
endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`
} else {
endpoint = `https://${bucketName}.s3.amazonaws.com`;
endpoint = `https://${bucketName}.s3.amazonaws.com`
}
return {
aws: {
@ -54,8 +54,8 @@ const buildDefaultOptions = function(bucketName, method, key){
method,
timeout: thirtySeconds,
uri: `${endpoint}/${key}`
};
};
}
}
const getS3Options = function(credentials) {
const options = {
@ -63,238 +63,314 @@ const getS3Options = function(credentials) {
accessKeyId: credentials.auth_key,
secretAccessKey: credentials.auth_secret
}
};
if (settings.filestore.s3.endpoint) {
const endpoint = URL.parse(settings.filestore.s3.endpoint);
options.endpoint = settings.filestore.s3.endpoint;
options.sslEnabled = endpoint.protocol === 'https';
}
return options;
};
if (settings.filestore.s3.endpoint) {
const endpoint = URL.parse(settings.filestore.s3.endpoint)
options.endpoint = settings.filestore.s3.endpoint
options.sslEnabled = endpoint.protocol === 'https'
}
const defaultS3Client = new awsS3(getS3Options({
return options
}
const defaultS3Client = new awsS3(
getS3Options({
auth_key: settings.filestore.s3.key,
auth_secret: settings.filestore.s3.secret
}));
})
)
const getS3Client = function(credentials) {
if (credentials != null) {
return new awsS3(getS3Options(credentials));
return new awsS3(getS3Options(credentials))
} else {
return defaultS3Client;
return defaultS3Client
}
}
};
const getKnoxClient = bucketName => {
const options = {
key: settings.filestore.s3.key,
secret: settings.filestore.s3.secret,
bucket: bucketName
};
if (settings.filestore.s3.endpoint) {
const endpoint = URL.parse(settings.filestore.s3.endpoint);
options.endpoint = endpoint.hostname;
options.port = endpoint.port;
}
return knox.createClient(options);
};
if (settings.filestore.s3.endpoint) {
const endpoint = URL.parse(settings.filestore.s3.endpoint)
options.endpoint = endpoint.hostname
options.port = endpoint.port
}
return knox.createClient(options)
}
module.exports = {
sendFile(bucketName, key, fsPath, callback) {
const s3Client = getKnoxClient(bucketName);
let uploaded = 0;
const s3Client = getKnoxClient(bucketName)
let uploaded = 0
const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) {
metrics.count('s3.egress', uploaded);
metrics.count('s3.egress', uploaded)
if (err != null) {
logger.err({err, bucketName, key, fsPath},"something went wrong uploading file to s3");
return callback(err);
logger.err(
{ err, bucketName, key, fsPath },
'something went wrong uploading file to s3'
)
return callback(err)
}
if ((res == null)) {
logger.err({err, res, bucketName, key, fsPath}, "no response from s3 put file");
return callback("no response from put file");
if (res == null) {
logger.err(
{ err, res, bucketName, key, fsPath },
'no response from s3 put file'
)
return callback('no response from put file')
}
if (res.statusCode !== 200) {
logger.err({bucketName, key, fsPath}, "non 200 response from s3 putting file");
return callback("non 200 response from s3 on put file");
logger.err(
{ bucketName, key, fsPath },
'non 200 response from s3 putting file'
)
return callback('non 200 response from s3 on put file')
}
logger.log({res, bucketName, key, fsPath},"file uploaded to s3");
return callback(err);
});
putEventEmiter.on("error", function(err){
logger.err({err, bucketName, key, fsPath}, "error emmited on put of file");
return callback(err);
});
return putEventEmiter.on("progress", progress => uploaded = progress.written);
logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3')
return callback(err)
})
putEventEmiter.on('error', function(err) {
logger.err(
{ err, bucketName, key, fsPath },
'error emmited on put of file'
)
return callback(err)
})
return putEventEmiter.on(
'progress',
progress => (uploaded = progress.written)
)
},
sendStream(bucketName, key, readStream, callback) {
logger.log({bucketName, key}, "sending file to s3");
readStream.on("error", err => logger.err({bucketName, key}, "error on stream to send to s3"));
logger.log({ bucketName, key }, 'sending file to s3')
readStream.on('error', err =>
logger.err({ bucketName, key }, 'error on stream to send to s3')
)
return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => {
if (err != null) {
logger.err({bucketName, key, fsPath, err}, "something went wrong writing stream to disk");
return callback(err);
logger.err(
{ bucketName, key, fsPath, err },
'something went wrong writing stream to disk'
)
return callback(err)
}
return this.sendFile(bucketName, key, fsPath, err => // delete the temporary file created above and return the original error
LocalFileWriter.deleteFile(fsPath, () => callback(err)));
});
return this.sendFile(bucketName, key, fsPath, (
err // delete the temporary file created above and return the original error
) => LocalFileWriter.deleteFile(fsPath, () => callback(err)))
})
},
// opts may be {start: Number, end: Number}
getFileStream(bucketName, key, opts, callback) {
if (callback == null) { callback = function(err, res){}; }
opts = opts || {};
callback = _.once(callback);
logger.log({bucketName, key}, "getting file from s3");
if (callback == null) {
callback = function(err, res) {}
}
opts = opts || {}
callback = _.once(callback)
logger.log({ bucketName, key }, 'getting file from s3')
const s3 = getS3Client(opts.credentials);
const s3 = getS3Client(opts.credentials)
const s3Params = {
Bucket: bucketName,
Key: key
};
if ((opts.start != null) && (opts.end != null)) {
s3Params.Range = `bytes=${opts.start}-${opts.end}`;
}
const s3Request = s3.getObject(s3Params);
if (opts.start != null && opts.end != null) {
s3Params.Range = `bytes=${opts.start}-${opts.end}`
}
const s3Request = s3.getObject(s3Params)
s3Request.on('httpHeaders', (statusCode, headers, response, statusMessage) => {
s3Request.on(
'httpHeaders',
(statusCode, headers, response, statusMessage) => {
if ([403, 404].includes(statusCode)) {
// S3 returns a 403 instead of a 404 when the user doesn't have
// permission to list the bucket contents.
logger.log({ bucketName, key }, "file not found in s3");
return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`), null);
logger.log({ bucketName, key }, 'file not found in s3')
return callback(
new Errors.NotFoundError(
`File not found in S3: ${bucketName}:${key}`
),
null
)
}
if (![200, 206].includes(statusCode)) {
logger.log({bucketName, key }, `error getting file from s3: ${statusCode}`);
return callback(new Error(`Got non-200 response from S3: ${statusCode} ${statusMessage}`), null);
logger.log(
{ bucketName, key },
`error getting file from s3: ${statusCode}`
)
return callback(
new Error(
`Got non-200 response from S3: ${statusCode} ${statusMessage}`
),
null
)
}
const stream = response.httpResponse.createUnbufferedStream();
stream.on('data', data => metrics.count('s3.ingress', data.byteLength));
const stream = response.httpResponse.createUnbufferedStream()
stream.on('data', data => metrics.count('s3.ingress', data.byteLength))
return callback(null, stream);
});
return callback(null, stream)
}
)
s3Request.on('error', err => {
logger.err({ err, bucketName, key }, "error getting file stream from s3");
return callback(err);
});
logger.err({ err, bucketName, key }, 'error getting file stream from s3')
return callback(err)
})
return s3Request.send();
return s3Request.send()
},
getFileSize(bucketName, key, callback) {
logger.log({ bucketName, key }, "getting file size from S3");
const s3 = getS3Client();
logger.log({ bucketName, key }, 'getting file size from S3')
const s3 = getS3Client()
return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
if (err != null) {
if ([403, 404].includes(err.statusCode)) {
// S3 returns a 403 instead of a 404 when the user doesn't have
// permission to list the bucket contents.
logger.log({
logger.log(
{
bucketName,
key
}, "file not found in s3");
},
'file not found in s3'
)
callback(
new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
);
new Errors.NotFoundError(
`File not found in S3: ${bucketName}:${key}`
)
)
} else {
logger.err({
logger.err(
{
bucketName,
key,
err
}, "error performing S3 HeadObject");
callback(err);
},
'error performing S3 HeadObject'
)
callback(err)
}
return;
return
}
return callback(null, data.ContentLength);
});
return callback(null, data.ContentLength)
})
},
copyFile(bucketName, sourceKey, destKey, callback) {
logger.log({bucketName, sourceKey, destKey}, "copying file in s3");
const source = bucketName + '/' + sourceKey;
logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
const source = bucketName + '/' + sourceKey
// use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
const s3 = getS3Client();
return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) {
const s3 = getS3Client()
return s3.copyObject(
{ Bucket: bucketName, Key: destKey, CopySource: source },
function(err) {
if (err != null) {
if (err.code === 'NoSuchKey') {
logger.err({bucketName, sourceKey}, "original file not found in s3 when copying");
return callback(new Errors.NotFoundError("original file not found in S3 when copying"));
logger.err(
{ bucketName, sourceKey },
'original file not found in s3 when copying'
)
return callback(
new Errors.NotFoundError(
'original file not found in S3 when copying'
)
)
} else {
logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in aws");
return callback(err);
logger.err(
{ err, bucketName, sourceKey, destKey },
'something went wrong copying file in aws'
)
return callback(err)
}
} else {
return callback();
return callback()
}
});
}
)
},
deleteFile(bucketName, key, callback) {
logger.log({bucketName, key}, "delete file in s3");
const options = buildDefaultOptions(bucketName, "delete", key);
logger.log({ bucketName, key }, 'delete file in s3')
const options = buildDefaultOptions(bucketName, 'delete', key)
return request(options, function(err, res) {
if (err != null) {
logger.err({err, res, bucketName, key}, "something went wrong deleting file in aws");
logger.err(
{ err, res, bucketName, key },
'something went wrong deleting file in aws'
)
}
return callback(err);
});
return callback(err)
})
},
deleteDirectory(bucketName, key, _callback) {
// deleteMultiple can call the callback multiple times so protect against this.
const callback = function(...args) {
_callback(...Array.from(args || []));
return _callback = function() {};
};
_callback(...Array.from(args || []))
return (_callback = function() {})
}
logger.log({key, bucketName}, "deleting directory");
const s3Client = getKnoxClient(bucketName);
logger.log({ key, bucketName }, 'deleting directory')
const s3Client = getKnoxClient(bucketName)
return s3Client.list({ prefix: key }, function(err, data) {
if (err != null) {
logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
return callback(err);
logger.err(
{ err, bucketName, key },
'something went wrong listing prefix in aws'
)
return callback(err)
}
const keys = _.map(data.Contents, entry => entry.Key);
return s3Client.deleteMultiple(keys, callback);
});
const keys = _.map(data.Contents, entry => entry.Key)
return s3Client.deleteMultiple(keys, callback)
})
},
checkIfFileExists(bucketName, key, callback) {
logger.log({bucketName, key}, "checking if file exists in s3");
const options = buildDefaultOptions(bucketName, "head", key);
logger.log({ bucketName, key }, 'checking if file exists in s3')
const options = buildDefaultOptions(bucketName, 'head', key)
return request(options, function(err, res) {
if (err != null) {
logger.err({err, res, bucketName, key}, "something went wrong checking file in aws");
return callback(err);
logger.err(
{ err, res, bucketName, key },
'something went wrong checking file in aws'
)
return callback(err)
}
if ((res == null)) {
logger.err({err, res, bucketName, key}, "no response object returned when checking if file exists");
err = new Error(`no response from s3 ${bucketName} ${key}`);
return callback(err);
if (res == null) {
logger.err(
{ err, res, bucketName, key },
'no response object returned when checking if file exists'
)
err = new Error(`no response from s3 ${bucketName} ${key}`)
return callback(err)
}
const exists = res.statusCode === 200;
logger.log({bucketName, key, exists}, "checked if file exsists in s3");
return callback(err, exists);
});
const exists = res.statusCode === 200
logger.log({ bucketName, key, exists }, 'checked if file exsists in s3')
return callback(err, exists)
})
},
directorySize(bucketName, key, callback) {
logger.log({bucketName, key}, "get project size in s3");
const s3Client = getKnoxClient(bucketName);
logger.log({ bucketName, key }, 'get project size in s3')
const s3Client = getKnoxClient(bucketName)
return s3Client.list({ prefix: key }, function(err, data) {
if (err != null) {
logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
return callback(err);
logger.err(
{ err, bucketName, key },
'something went wrong listing prefix in aws'
)
return callback(err)
}
let totalSize = 0
_.each(data.Contents, entry => (totalSize += entry.Size))
logger.log({ totalSize }, 'total size')
return callback(null, totalSize)
})
}
let totalSize = 0;
_.each(data.Contents, entry => totalSize += entry.Size);
logger.log({totalSize}, "total size");
return callback(null, totalSize);
});
}
};

56
services/filestore/app/js/SafeExec.js
View file

@ -12,10 +12,10 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const _ = require("underscore");
const logger = require("logger-sharelatex");
const child_process = require('child_process');
const Settings = require("settings-sharelatex");
const _ = require('underscore')
const logger = require('logger-sharelatex')
const child_process = require('child_process')
const Settings = require('settings-sharelatex')
// execute a command in the same way as 'exec' but with a timeout that
// kills all child processes
@ -24,44 +24,50 @@ const Settings = require("settings-sharelatex");
// group, then we can kill everything in that process group.
module.exports = function(command, options, callback) {
if (callback == null) { callback = function(err, stdout, stderr) {}; }
if (callback == null) {
callback = function(err, stdout, stderr) {}
}
if (!Settings.enableConversions) {
const error = new Error("Image conversions are disabled");
return callback(error);
const error = new Error('Image conversions are disabled')
return callback(error)
}
// options are {timeout: number-of-milliseconds, killSignal: signal-name}
const [cmd, ...args] = Array.from(command);
const [cmd, ...args] = Array.from(command)
const child = child_process.spawn(cmd, args, {detached:true});
let stdout = "";
let stderr = "";
const child = child_process.spawn(cmd, args, { detached: true })
let stdout = ''
let stderr = ''
const cleanup = _.once(function(err) {
if (killTimer != null) { clearTimeout(killTimer); }
return callback(err, stdout, stderr);
});
if (killTimer != null) {
clearTimeout(killTimer)
}
return callback(err, stdout, stderr)
})
if (options.timeout != null) {
var killTimer = setTimeout(function() {
try {
// use negative process id to kill process group
return process.kill(-child.pid, options.killSignal || "SIGTERM");
return process.kill(-child.pid, options.killSignal || 'SIGTERM')
} catch (error) {
return logger.log({process: child.pid, kill_error: error}, "error killing process");
return logger.log(
{ process: child.pid, kill_error: error },
'error killing process'
)
}
}
, options.timeout);
}, options.timeout)
}
child.on('close', function(code, signal) {
const err = code ? new Error(`exit status ${code}`) : signal;
return cleanup(err);
});
const err = code ? new Error(`exit status ${code}`) : signal
return cleanup(err)
})
child.on('error', err => cleanup(err));
child.on('error', err => cleanup(err))
child.stdout.on('data', chunk => stdout += chunk);
child.stdout.on('data', chunk => (stdout += chunk))
return child.stderr.on('data', chunk => stderr += chunk);
};
return child.stderr.on('data', chunk => (stderr += chunk))
}