From 0db4a17a143198c7eb6f90ad9d9815028347c64e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 10 Jan 2020 01:13:32 +0100 Subject: [PATCH 1/5] [HealthCheckController] use fs.copyFile instead of fs-extra.copy the fs-extra method has a HUGE overhead of JS code and also syscalls for no particular benefit in this case: just copy the tiny.pdf file. Here is an overview of the major operations: paths are relative to https://github.com/jprichardson/node-fs-extra/blob/1.0.0 We start in /lib/copy/copy.js - sys: check that the source file exists - sys: check that the source file has an existing parent directory?! Continue in /lib/copy/ncp.js - sys: more stat calls on both source and dest to determine permissions - read/write streams to pipe the file content through the process - sys: chmod on the destination to match the source permissions What we actually need is a call to the binding and let node/the os figure out the best way to copy the contents. Signed-off-by: Jakob Ackermann --- .../filestore/app/js/HealthCheckController.js | 4 +-- services/filestore/package-lock.json | 29 ++----------------- services/filestore/package.json | 1 - 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js index a52d02a444..0a4b10387e 100644 --- a/services/filestore/app/js/HealthCheckController.js +++ b/services/filestore/app/js/HealthCheckController.js @@ -1,4 +1,4 @@ -const fs = require('fs-extra') +const fs = require('fs') const path = require('path') const Settings = require('settings-sharelatex') const streamBuffers = require('stream-buffers') @@ -6,7 +6,7 @@ const { promisify } = require('util') const Stream = require('stream') const pipeline = promisify(Stream.pipeline) -const fsCopy = promisify(fs.copy) +const fsCopy = promisify(fs.copyFile) const fsUnlink = promisify(fs.unlink) const { HealthCheckError } = require('./Errors') diff --git a/services/filestore/package-lock.json b/services/filestore/package-lock.json index f8b4a70297..2fd623af1c 100644 --- a/services/filestore/package-lock.json +++ b/services/filestore/package-lock.json @@ -2488,16 +2488,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, - "fs-extra": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", - "integrity": "sha512-VerQV6vEKuhDWD2HGOybV6v5I73syoc/cXAbKlgTC7M/oFVEtklWlp9QH2Ijw3IaWDOQcMkldSPa7zXy79Z/UQ==", - "requires": { - "graceful-fs": "^4.1.2", - "jsonfile": "^2.1.0", - "klaw": "^1.0.0" - } - }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2644,7 +2634,8 @@ "graceful-fs": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", - "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", + "dev": true }, "growl": { "version": "1.10.5", @@ -3072,14 +3063,6 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, - "jsonfile": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", - "integrity": "sha512-PKllAqbgLgxHaj8TElYymKCAgrASebJrWpTnEkOaTowt23VKXXN0sUeriJ+eh7y6ufb/CC5ap11pz71/cM0hUw==", - "requires": { - "graceful-fs": "^4.1.6" - } - }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -3116,14 +3099,6 @@ "safe-buffer": "^5.0.1" } }, - "klaw": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", - "integrity": "sha512-TED5xi9gGQjGpNnvRWknrwAB1eL5GciPfVFOt3Vk1OJCVDQbzuSfrF3hkUQKlsgKrG1F+0t5W0m+Fje1jIt8rw==", - "requires": { - "graceful-fs": "^4.1.9" - } - }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index 4a5d72abb5..45d7eed1c3 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -25,7 +25,6 @@ "aws-sdk": "^2.628.0", "body-parser": "^1.2.0", "express": "^4.2.0", - "fs-extra": "^1.0.0", "glob": "^7.1.6", "heapdump": "^0.3.2", "logger-sharelatex": "^1.7.0", From 32557ab1d7fe941a54a99afe4acd95dd600ce5f2 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 10 Jan 2020 10:18:55 +0000 Subject: [PATCH 2/5] [SafeExec] replace _.once with lodash.once --- services/filestore/app/js/SafeExec.js | 4 ++-- services/filestore/package-lock.json | 10 +++++----- services/filestore/package.json | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/filestore/app/js/SafeExec.js b/services/filestore/app/js/SafeExec.js index 5ed0f18425..a9d1398441 100644 --- a/services/filestore/app/js/SafeExec.js +++ b/services/filestore/app/js/SafeExec.js @@ -1,4 +1,4 @@ -const _ = require('underscore') +const lodashOnce = require('lodash.once') const childProcess = require('child_process') const Settings = require('settings-sharelatex') const { ConversionsDisabledError, FailedCommandError } = require('./Errors') @@ -28,7 +28,7 @@ function safeExec(command, options, callback) { let killTimer - const cleanup = _.once(function(err) { + const cleanup = lodashOnce(function(err) { if (killTimer) { clearTimeout(killTimer) } diff --git a/services/filestore/package-lock.json b/services/filestore/package-lock.json index 2fd623af1c..6356c89d30 100644 --- a/services/filestore/package-lock.json +++ b/services/filestore/package-lock.json @@ -3177,6 +3177,11 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, "lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", @@ -5478,11 +5483,6 @@ "integrity": "sha512-EgOVgL/4xfVrCMbhYKUQTdF37SQn4Iw73H5BgCrF1Abdun7Kwy/QZsE/ssAy0y4LxBbvua3PIbFsbRczWWnDdQ==", "dev": true }, - "underscore": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.5.2.tgz", - "integrity": "sha512-yejOFsRnTJs0N9CK5Apzf6maDO2djxGoLLrlZlvGs2o9ZQuhIhDL18rtFyy4FBIbOkzA6+4hDgXbgz5EvDQCXQ==" - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index 45d7eed1c3..ad7fd42bb6 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -27,6 +27,7 @@ "express": "^4.2.0", "glob": "^7.1.6", "heapdump": "^0.3.2", + "lodash.once": "^4.1.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", "mocha": "5.2.0", @@ -38,8 +39,7 @@ "rimraf": "2.2.8", "settings-sharelatex": "^1.1.0", "stream-buffers": "~0.2.5", - "stream-meter": "^1.0.4", - "underscore": "~1.5.2" + "stream-meter": "^1.0.4" }, "devDependencies": { "babel-eslint": "^10.0.3", From 6f27f7a1bffcb5dde5da4adb57091038692d543a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sun, 23 Feb 2020 16:27:55 +0100 Subject: [PATCH 3/5] [misc] drop the /heapdump route and related heapdump package --- services/filestore/README.md | 1 - services/filestore/app.js | 12 ------------ services/filestore/package-lock.json | 8 -------- services/filestore/package.json | 1 - 4 files changed, 22 deletions(-) diff --git a/services/filestore/README.md b/services/filestore/README.md index 2772b71494..3ee6cadff6 100644 --- a/services/filestore/README.md +++ b/services/filestore/README.md @@ -10,7 +10,6 @@ filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, p * `/project/:project_id/public/:public_file_id` * `/project/:project_id/size` * `/bucket/:bucket/key/*` -* `/heapdump` * `/shutdown` * `/status` - returns `filestore sharelatex up` or `server is being shut down` (HTTP 500) * `/health_check` diff --git a/services/filestore/app.js b/services/filestore/app.js index 278997b9aa..ea2c2ca1d8 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -122,18 +122,6 @@ app.get( fileController.getFile ) -app.get('/heapdump', (req, res, next) => - require('heapdump').writeSnapshot( - '/tmp/' + Date.now() + '.filestore.heapsnapshot', - (err, filename) => { - if (err) { - return next(err) - } - res.send(filename) - } - ) -) - app.get('/status', function(req, res) { res.send('filestore sharelatex up') }) diff --git a/services/filestore/package-lock.json b/services/filestore/package-lock.json index 6356c89d30..fa85d92a20 100644 --- a/services/filestore/package-lock.json +++ b/services/filestore/package-lock.json @@ -2716,14 +2716,6 @@ "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", "integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==" }, - "heapdump": { - "version": "0.3.15", - "resolved": "https://registry.npmjs.org/heapdump/-/heapdump-0.3.15.tgz", - "integrity": "sha512-n8aSFscI9r3gfhOcAECAtXFaQ1uy4QSke6bnaL+iymYZ/dWs9cqDqHM+rALfsHUwukUbxsdlECZ0pKmJdQ/4OA==", - "requires": { - "nan": "^2.13.2" - } - }, "hex2dec": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index ad7fd42bb6..c40f2a65f6 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -26,7 +26,6 @@ "body-parser": "^1.2.0", "express": "^4.2.0", "glob": "^7.1.6", - "heapdump": "^0.3.2", "lodash.once": "^4.1.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", From 3c61e53918026545e1313dfa5cb590dde6c5090a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sun, 23 Feb 2020 18:41:14 +0000 Subject: [PATCH 4/5] [misc] move mocha to the devDependencies --- services/filestore/package-lock.json | 25 ++++++++++++++++++------- services/filestore/package.json | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/services/filestore/package-lock.json b/services/filestore/package-lock.json index fa85d92a20..12b57d5975 100644 --- a/services/filestore/package-lock.json +++ b/services/filestore/package-lock.json @@ -1409,7 +1409,8 @@ "browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", - "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true }, "buffer": { "version": "4.9.1", @@ -1603,7 +1604,8 @@ "commander": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", - "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==" + "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", + "dev": true }, "common-tags": { "version": "1.8.0", @@ -1765,7 +1767,8 @@ "diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==" + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true }, "disrequire": { "version": "1.1.0", @@ -1930,7 +1933,8 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true }, "eslint": { "version": "6.8.0", @@ -2640,7 +2644,8 @@ "growl": { "version": "1.10.5", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==" + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true }, "gtoken": { "version": "4.1.4", @@ -2703,7 +2708,8 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true }, "has-symbols": { "version": "1.0.1", @@ -2714,7 +2720,8 @@ "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==" + "integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==", + "dev": true }, "hex2dec": { "version": "1.1.2", @@ -3470,6 +3477,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", + "dev": true, "requires": { "browser-stdout": "1.3.1", "commander": "2.15.1", @@ -3488,6 +3496,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "dev": true, "requires": { "ms": "2.0.0" } @@ -3496,6 +3505,7 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -5277,6 +5287,7 @@ "version": "5.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", + "dev": true, "requires": { "has-flag": "^3.0.0" } diff --git a/services/filestore/package.json b/services/filestore/package.json index c40f2a65f6..509e4f962a 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -29,7 +29,6 @@ "lodash.once": "^4.1.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", - "mocha": "5.2.0", "node-uuid": "~1.4.1", "pngcrush": "0.0.3", "range-parser": "^1.0.2", @@ -57,6 +56,7 @@ "eslint-plugin-prettier": "^3.1.2", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", + "mocha": "5.2.0", "prettier-eslint": "^9.0.1", "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "2.0.3", From 4b7c6fafd2d053c689747980d3b6c915bbcc399d Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sun, 23 Feb 2020 21:51:14 +0000 Subject: [PATCH 5/5] [misc] drop unused dependency pngcrush --- services/filestore/package-lock.json | 13 ------------- services/filestore/package.json | 1 - 2 files changed, 14 deletions(-) diff --git a/services/filestore/package-lock.json b/services/filestore/package-lock.json index 12b57d5975..c3ed1ad560 100644 --- a/services/filestore/package-lock.json +++ b/services/filestore/package-lock.json @@ -2556,11 +2556,6 @@ "assert-plus": "^1.0.0" } }, - "gettemporaryfilepath": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/gettemporaryfilepath/-/gettemporaryfilepath-0.0.1.tgz", - "integrity": "sha512-7avwQWP8MP42u7mtc+KjCRuUE3nafRJPuGaZaySD9NN1KEbfVTfSAywP4KOkK8gaxhdOxx11ZTWH28DwjAF70Q==" - }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -3909,14 +3904,6 @@ "find-up": "^2.1.0" } }, - "pngcrush": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/pngcrush/-/pngcrush-0.0.3.tgz", - "integrity": "sha512-RVaPWGv0PUUzGeSQJHH78rw2ks8NxKbFn8uENFM+/3bfsUs39MaFDG+eul5902gH97zZLQ0zd0h2yb0YBaMKDw==", - "requires": { - "gettemporaryfilepath": "=0.0.1" - } - }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index 509e4f962a..04774dc5e4 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -30,7 +30,6 @@ "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", "node-uuid": "~1.4.1", - "pngcrush": "0.0.3", "range-parser": "^1.0.2", "request": "^2.88.0", "request-promise-native": "^1.0.8",