From 12e7471213c7c1ad38078c1b3aa79cbc1c203045 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween <5454374+emcsween@users.noreply.github.com> Date: Thu, 1 Jun 2023 07:38:45 -0400 Subject: [PATCH] Merge pull request #12916 from overleaf/bg-move-stream-buffer-code-to-library move stream-related code to separate `@overleaf/stream-utils` library GitOrigin-RevId: a79a873109b927b4fc0ae36f47d5c67e0df58041 --- libraries/stream-utils/.dockerignore | 1 + libraries/stream-utils/.gitignore | 3 + libraries/stream-utils/.mocharc.json | 5 + libraries/stream-utils/.nvmrc | 1 + libraries/stream-utils/Dockerfile | 5 + libraries/stream-utils/buildscript.txt | 9 + libraries/stream-utils/index.js | 158 +++++++++ libraries/stream-utils/package.json | 22 ++ .../test/unit/LimitedStreamTests.js | 30 ++ .../test/unit/LoggerStreamTests.js | 36 ++ .../test/unit/ReadableStringTests.js | 16 + .../test/unit/TimeoutStreamTests.js | 22 ++ .../test/unit/WritableBufferTests.js | 20 ++ package-lock.json | 327 ++++++------------ services/docstore/app/js/DocArchiveManager.js | 4 +- services/docstore/package.json | 4 +- .../test/acceptance/js/ArchiveDocsTests.js | 4 +- .../test/unit/js/DocArchiveManagerTests.js | 12 +- .../filestore/app/js/HealthCheckController.js | 6 +- services/filestore/package.json | 2 +- .../test/acceptance/js/FilestoreTests.js | 18 +- services/history-v1/package.json | 2 +- .../storage/lib/blob_store/index.js | 6 +- services/history-v1/storage/lib/streams.js | 26 +- services/project-history/package.json | 2 - services/web/package.json | 1 - .../ThirdPartyDataStore/UpdateMergerTests.js | 4 +- 27 files changed, 467 insertions(+), 279 deletions(-) create mode 100644 libraries/stream-utils/.dockerignore create mode 100644 libraries/stream-utils/.gitignore create mode 100644 libraries/stream-utils/.mocharc.json create mode 100644 libraries/stream-utils/.nvmrc create mode 100644 libraries/stream-utils/Dockerfile create mode 100644 libraries/stream-utils/buildscript.txt create mode 100644 libraries/stream-utils/index.js create mode 100644 libraries/stream-utils/package.json create mode 100644 libraries/stream-utils/test/unit/LimitedStreamTests.js create mode 100644 libraries/stream-utils/test/unit/LoggerStreamTests.js create mode 100644 libraries/stream-utils/test/unit/ReadableStringTests.js create mode 100644 libraries/stream-utils/test/unit/TimeoutStreamTests.js create mode 100644 libraries/stream-utils/test/unit/WritableBufferTests.js diff --git a/libraries/stream-utils/.dockerignore b/libraries/stream-utils/.dockerignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/libraries/stream-utils/.dockerignore @@ -0,0 +1 @@ +node_modules/ diff --git a/libraries/stream-utils/.gitignore b/libraries/stream-utils/.gitignore new file mode 100644 index 0000000000..edb0f85350 --- /dev/null +++ b/libraries/stream-utils/.gitignore @@ -0,0 +1,3 @@ + +# managed by monorepo$ bin/update_build_scripts +.npmrc diff --git a/libraries/stream-utils/.mocharc.json b/libraries/stream-utils/.mocharc.json new file mode 100644 index 0000000000..3be9ee53ae --- /dev/null +++ b/libraries/stream-utils/.mocharc.json @@ -0,0 +1,5 @@ +{ + "ui": "bdd", + "recursive": "true", + "reporter": "spec" +} diff --git a/libraries/stream-utils/.nvmrc b/libraries/stream-utils/.nvmrc new file mode 100644 index 0000000000..c85fa1bbef --- /dev/null +++ b/libraries/stream-utils/.nvmrc @@ -0,0 +1 @@ +16.17.1 diff --git a/libraries/stream-utils/Dockerfile b/libraries/stream-utils/Dockerfile new file mode 100644 index 0000000000..5c971cea64 --- /dev/null +++ b/libraries/stream-utils/Dockerfile @@ -0,0 +1,5 @@ +FROM node:16.17.1 + +WORKDIR /app + +USER node diff --git a/libraries/stream-utils/buildscript.txt b/libraries/stream-utils/buildscript.txt new file mode 100644 index 0000000000..6fa0482537 --- /dev/null +++ b/libraries/stream-utils/buildscript.txt @@ -0,0 +1,9 @@ +stream-utils +--dependencies=None +--docker-repos=gcr.io/overleaf-ops +--env-add= +--env-pass-through= +--is-library=True +--node-version=16.17.1 +--public-repo=False +--script-version=4.1.0 diff --git a/libraries/stream-utils/index.js b/libraries/stream-utils/index.js new file mode 100644 index 0000000000..274dc0b378 --- /dev/null +++ b/libraries/stream-utils/index.js @@ -0,0 +1,158 @@ +const { Writable, Readable, PassThrough, Transform } = require('stream') + +/** + * A writable stream that stores all data written to it in a node Buffer. + * @extends stream.Writable + * @example + * const { WritableBuffer } = require('@overleaf/stream-utils') + * const bufferStream = new WritableBuffer() + * bufferStream.write('hello') + * bufferStream.write('world') + * bufferStream.end() + * bufferStream.contents().toString() // 'helloworld' + */ +class WritableBuffer extends Writable { + constructor(options) { + super(options) + this._buffers = [] + this._size = 0 + } + + _write(chunk, encoding, callback) { + this._buffers.push(chunk) + this._size += chunk.length + callback() + } + + _final(callback) { + callback() + } + + size() { + return this._size + } + + getContents() { + return Buffer.concat(this._buffers) + } + + contents() { + return Buffer.concat(this._buffers) + } +} + +/** + * A readable stream created from a string. + * @extends stream.Readable + * @example + * const { ReadableString } = require('@overleaf/stream-utils') + * const stringStream = new ReadableString('hello world') + * stringStream.on('data', chunk => console.log(chunk.toString())) + * stringStream.on('end', () => console.log('done')) + */ +class ReadableString extends Readable { + constructor(string, options) { + super(options) + this._string = string + } + + _read(size) { + this.push(this._string) + this.push(null) + } +} + +class SizeExceededError extends Error {} + +/** + * Limited size stream which will emit a SizeExceededError if the size is exceeded + * @extends stream.Transform + */ +class LimitedStream extends Transform { + constructor(maxSize) { + super() + this.maxSize = maxSize + this.size = 0 + } + + _transform(chunk, encoding, callback) { + this.size += chunk.byteLength + if (this.size > this.maxSize) { + callback( + new SizeExceededError( + `exceeded stream size limit of ${this.maxSize}: ${this.size}` + ) + ) + } else { + callback(null, chunk) + } + } +} + +class AbortError extends Error {} + +/** + * TimeoutStream which will emit an AbortError if it exceeds a user specified timeout + * @extends stream.PassThrough + */ +class TimeoutStream extends PassThrough { + constructor(timeout) { + super() + this.t = setTimeout(() => { + this.destroy(new AbortError('stream timed out')) + }, timeout) + } + + _final(callback) { + clearTimeout(this.t) + callback() + } +} + +/** + * LoggerStream which will call the provided logger function when the stream exceeds a user specified limit. It will call the provided function again when flushing the stream and it exceeded the user specified limit before. +* @extends stream.Transform + */ +class LoggerStream extends Transform { + /** + * Constructor. + * @param {number} maxSize + * @param {function(currentSizeOfStream: number, isFlush: boolean)} fn + * @param {Object?} options optional options for the Transform stream + */ + constructor(maxSize, fn, options) { + super(options) + this.fn = fn + this.size = 0 + this.maxSize = maxSize + this.logged = false + } + + _transform(chunk, encoding, callback) { + this.size += chunk.byteLength + if (this.size > this.maxSize && !this.logged) { + this.fn(this.size) + this.logged = true + } + callback(null, chunk) + } + + _flush(callback) { + if (this.size > this.maxSize) { + this.fn(this.size, true) + } + callback() + } +} + +// Export our classes + +module.exports = { + WritableBuffer, + ReadableString, + LoggerStream, + LimitedStream, + TimeoutStream, + SizeExceededError, + AbortError, +} diff --git a/libraries/stream-utils/package.json b/libraries/stream-utils/package.json new file mode 100644 index 0000000000..f9beee688f --- /dev/null +++ b/libraries/stream-utils/package.json @@ -0,0 +1,22 @@ +{ + "name": "@overleaf/stream-utils", + "version": "0.1.0", + "description": "stream handling utilities", + "main": "index.js", + "scripts": { + "test": "npm run lint && npm run format && npm run test:unit", + "test:unit": "mocha", + "lint": "eslint --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix .", + "format": "prettier --list-different $PWD/'**/*.js'", + "format:fix": "prettier --write $PWD/'**/*.js'", + "test:ci": "npm run test:unit" + }, + "author": "Overleaf (https://www.overleaf.com)", + "license": "AGPL-3.0-only", + "devDependencies":{ + "chai": "^4.3.6", + "chai-as-promised": "^7.1.1", + "mocha": "^10.2.0" + } +} diff --git a/libraries/stream-utils/test/unit/LimitedStreamTests.js b/libraries/stream-utils/test/unit/LimitedStreamTests.js new file mode 100644 index 0000000000..6401b7de3e --- /dev/null +++ b/libraries/stream-utils/test/unit/LimitedStreamTests.js @@ -0,0 +1,30 @@ +const { expect } = require('chai') +const { LimitedStream, SizeExceededError } = require('../../index') + +describe('LimitedStream', function () { + it('should emit an error if the stream size exceeds the limit', function (done) { + const maxSize = 10 + const limitedStream = new LimitedStream(maxSize) + limitedStream.on('error', err => { + expect(err).to.be.an.instanceOf(SizeExceededError) + done() + }) + limitedStream.write(Buffer.alloc(maxSize + 1)) + }) + + it('should pass through data if the stream size does not exceed the limit', function (done) { + const maxSize = 15 + const limitedStream = new LimitedStream(maxSize) + let data = '' + limitedStream.on('data', chunk => { + data += chunk.toString() + }) + limitedStream.on('end', () => { + expect(data).to.equal('hello world') + done() + }) + limitedStream.write('hello') + limitedStream.write(' world') + limitedStream.end() + }) +}) diff --git a/libraries/stream-utils/test/unit/LoggerStreamTests.js b/libraries/stream-utils/test/unit/LoggerStreamTests.js new file mode 100644 index 0000000000..ed664f8823 --- /dev/null +++ b/libraries/stream-utils/test/unit/LoggerStreamTests.js @@ -0,0 +1,36 @@ +const { expect } = require('chai') +const { LoggerStream } = require('../../index') + +describe('LoggerStream', function () { + it('should log the size of the stream when it exceeds the limit', function (done) { + const maxSize = 10 + const loggedSizes = [] + const loggerStream = new LoggerStream(maxSize, (size, isFlush) => { + loggedSizes.push([size, isFlush]) + if (isFlush) { + expect(loggedSizes).to.deep.equal([ + [11, undefined], + [11, true], + ]) + done() + } + }) + loggerStream.write(Buffer.alloc(maxSize)) + loggerStream.write(Buffer.alloc(1)) + loggerStream.end() + }) + + it('should not log the size of the stream if it does not exceed the limit', function (done) { + const maxSize = 10 + const loggedSizes = [] + const loggerStream = new LoggerStream(maxSize, (size, isFlush) => { + loggedSizes.push(size) + }) + loggerStream.write(Buffer.alloc(maxSize)) + loggerStream.end() + loggerStream.on('finish', () => { + expect(loggedSizes).to.deep.equal([]) + done() + }) + }) +}) diff --git a/libraries/stream-utils/test/unit/ReadableStringTests.js b/libraries/stream-utils/test/unit/ReadableStringTests.js new file mode 100644 index 0000000000..b08dbc6806 --- /dev/null +++ b/libraries/stream-utils/test/unit/ReadableStringTests.js @@ -0,0 +1,16 @@ +const { expect } = require('chai') +const { ReadableString } = require('../../index') + +describe('ReadableString', function () { + it('should emit the string passed to it', function (done) { + const stringStream = new ReadableString('hello world') + let data = '' + stringStream.on('data', chunk => { + data += chunk.toString() + }) + stringStream.on('end', () => { + expect(data).to.equal('hello world') + done() + }) + }) +}) diff --git a/libraries/stream-utils/test/unit/TimeoutStreamTests.js b/libraries/stream-utils/test/unit/TimeoutStreamTests.js new file mode 100644 index 0000000000..2b203c5cc6 --- /dev/null +++ b/libraries/stream-utils/test/unit/TimeoutStreamTests.js @@ -0,0 +1,22 @@ +const { expect } = require('chai') +const { TimeoutStream, AbortError } = require('../../index') + +describe('TimeoutStream', function () { + it('should emit an error if the stream times out', function (done) { + const timeout = 10 + const timeoutStream = new TimeoutStream(timeout) + timeoutStream.on('error', err => { + expect(err).to.be.an.instanceOf(AbortError) + done() + }) + }) + + it('should not emit an error if the stream does not time out', function (done) { + const timeout = 100 + const timeoutStream = new TimeoutStream(timeout) + setTimeout(() => { + timeoutStream.end() + done() + }, 1) + }) +}) diff --git a/libraries/stream-utils/test/unit/WritableBufferTests.js b/libraries/stream-utils/test/unit/WritableBufferTests.js new file mode 100644 index 0000000000..a9b16a8f2c --- /dev/null +++ b/libraries/stream-utils/test/unit/WritableBufferTests.js @@ -0,0 +1,20 @@ +const { expect } = require('chai') +const { WritableBuffer } = require('../../index') + +describe('WritableBuffer', function () { + it('should store all data written to it in a node Buffer', function () { + const bufferStream = new WritableBuffer() + bufferStream.write('hello') + bufferStream.write('world') + bufferStream.end() + expect(bufferStream.contents().toString()).to.equal('helloworld') + }) + + it('should return the size of the data written to it', function () { + const bufferStream = new WritableBuffer() + bufferStream.write('hello') + bufferStream.write('world') + bufferStream.end() + expect(bufferStream.size()).to.equal(10) + }) +}) diff --git a/package-lock.json b/package-lock.json index 9b74b09587..b3da4ac98d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -377,6 +377,43 @@ "name": "@overleaf/settings", "version": "3.0.0" }, + "libraries/stream-utils": { + "name": "@overleaf/stream-utils", + "version": "0.1.0", + "license": "AGPL-3.0-only", + "dependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0" + } + }, + "libraries/stream-utils/node_modules/chai": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz", + "integrity": "sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==", + "dependencies": { + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^4.1.2", + "get-func-name": "^2.0.0", + "loupe": "^2.3.1", + "pathval": "^1.1.1", + "type-detect": "^4.0.5" + }, + "engines": { + "node": ">=4" + } + }, + "libraries/stream-utils/node_modules/deep-eql": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz", + "integrity": "sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==", + "dependencies": { + "type-detect": "^4.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -7987,6 +8024,10 @@ "resolved": "services/spelling", "link": true }, + "node_modules/@overleaf/stream-utils": { + "resolved": "libraries/stream-utils", + "link": true + }, "node_modules/@overleaf/templates": { "resolved": "services/templates", "link": true @@ -10790,7 +10831,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", - "dev": true, "engines": { "node": ">=6" } @@ -10911,7 +10951,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "devOptional": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -11226,7 +11265,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", - "dev": true, "engines": { "node": "*" } @@ -11863,7 +11901,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "devOptional": true, "engines": { "node": ">=8" } @@ -12086,8 +12123,7 @@ "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", - "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", - "dev": true + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" }, "node_modules/browserslist": { "version": "4.21.5", @@ -12193,14 +12229,6 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, - "node_modules/bufferedstream": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/bufferedstream/-/bufferedstream-1.6.0.tgz", - "integrity": "sha1-G1a+ZxMhYtn0aLyIbdLJFg3fKII=", - "engines": { - "node": ">=0.4.7" - } - }, "node_modules/buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", @@ -12841,7 +12869,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", - "dev": true, "engines": { "node": "*" } @@ -12901,7 +12928,6 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "devOptional": true, "funding": [ { "type": "individual", @@ -15859,7 +15885,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", - "dev": true, "engines": { "node": ">=0.3.1" } @@ -18985,7 +19010,6 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", - "dev": true, "bin": { "flat": "cli.js" } @@ -19381,7 +19405,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -19537,7 +19560,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", - "dev": true, "engines": { "node": "*" } @@ -19657,7 +19679,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "devOptional": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -20652,7 +20673,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true, "bin": { "he": "bin/he" } @@ -21570,7 +21590,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "devOptional": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -21681,7 +21700,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", - "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -21717,7 +21735,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -22016,7 +22033,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "dev": true, "engines": { "node": ">=10" }, @@ -24630,7 +24646,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, "dependencies": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" @@ -24646,7 +24661,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -24661,7 +24675,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -24677,7 +24690,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -24778,7 +24790,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.1.tgz", "integrity": "sha512-EN1D3jyVmaX4tnajVlfbREU4axL647hLec1h/PXAb8CPDMJiYitcWF2UeLVNttRqaIqQs4x+mRvXf+d+TlDrCA==", - "dev": true, "dependencies": { "get-func-name": "^2.0.0" } @@ -25575,7 +25586,6 @@ "version": "10.2.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", - "dev": true, "dependencies": { "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", @@ -25624,7 +25634,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -25633,7 +25642,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -25649,14 +25657,12 @@ "node_modules/mocha/node_modules/debug/node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mocha/node_modules/minimatch": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", - "dev": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -25668,7 +25674,6 @@ "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dev": true, "dependencies": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -26081,7 +26086,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true, "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -27763,7 +27767,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", - "dev": true, "engines": { "node": "*" } @@ -29421,7 +29424,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "devOptional": true, "dependencies": { "picomatch": "^2.2.1" }, @@ -30816,7 +30818,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", - "dev": true, "dependencies": { "randombytes": "^2.1.0" } @@ -30825,7 +30826,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, "dependencies": { "safe-buffer": "^5.1.0" } @@ -31746,14 +31746,6 @@ "node": ">= 0.4" } }, - "node_modules/stream-buffers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz", - "integrity": "sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==", - "engines": { - "node": ">= 0.10.0" - } - }, "node_modules/stream-connect": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/stream-connect/-/stream-connect-1.0.2.tgz", @@ -31812,6 +31804,7 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/streamifier/-/streamifier-0.1.1.tgz", "integrity": "sha1-l+mNj6TRBdYqJpHR3AfoINuN/E8=", + "dev": true, "engines": { "node": ">=0.10" } @@ -31875,42 +31868,6 @@ "resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz", "integrity": "sha512-Yptehjogou2xm4UJbxJ4CxgZx12HBfeystp0y3x7s4Dj32ltVVG1Gg8YhKjHZkHicuKpZX/ffilA8505VbUbpw==" }, - "node_modules/string-to-stream": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz", - "integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==", - "dependencies": { - "inherits": "^2.0.1", - "readable-stream": "^2.1.0" - } - }, - "node_modules/string-to-stream/node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/string-to-stream/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/string-to-stream/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -33470,7 +33427,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, "engines": { "node": ">=4" } @@ -35187,7 +35143,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", - "dev": true, "dependencies": { "camelcase": "^6.0.0", "decamelize": "^4.0.0", @@ -35202,7 +35157,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", - "dev": true, "engines": { "node": ">=10" }, @@ -35214,7 +35168,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", - "dev": true, "engines": { "node": ">=8" } @@ -35350,6 +35303,7 @@ "@overleaf/o-error": "*", "@overleaf/redis-wrapper": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "axios": "^0.21.1", "bluebird": "^3.7.2", "body-parser": "^1.19.0", @@ -35376,7 +35330,6 @@ "recurly": "^4.0.1", "request": "^2.88.2", "sequelize": "^6.31.0", - "stream-buffers": "^3.0.2", "underscore": "^1.9.2", "yargs": "^17.0.0" }, @@ -35724,6 +35677,7 @@ "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "async": "^3.2.2", "body-parser": "^1.19.0", "bson": "^1.1.4", @@ -35733,8 +35687,7 @@ "lodash": "^4.17.21", "mongodb": "^4.11.0", "p-map": "^4.0.0", - "request": "^2.88.2", - "streamifier": "^0.1.1" + "request": "^2.88.2" }, "devDependencies": { "@google-cloud/storage": "^6.10.1", @@ -35944,6 +35897,7 @@ "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "body-parser": "^1.19.0", "bunyan": "^1.8.15", "express": "^4.18.2", @@ -35951,7 +35905,6 @@ "lodash.once": "^4.1.1", "node-fetch": "^2.6.7", "range-parser": "^1.2.1", - "stream-buffers": "~0.2.6", "tiny-async-pool": "^1.1.0" }, "devDependencies": { @@ -35997,14 +35950,6 @@ "url": "https://opencollective.com/sinon" } }, - "services/filestore/node_modules/stream-buffers": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-0.2.6.tgz", - "integrity": "sha1-GBwI1bs2kARfaUAbmuanoM8zE/w=", - "engines": { - "node": ">= 0.3.0" - } - }, "services/filestore/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -36141,6 +36086,7 @@ "@overleaf/metrics": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", + "@overleaf/stream-utils": "^0.1.0", "archiver": "^5.3.0", "basic-auth": "^2.0.1", "bluebird": "^3.7.2", @@ -36162,7 +36108,6 @@ "mongodb": "^4.11.0", "overleaf-editor-core": "*", "pg": "^8.7.1", - "string-to-stream": "^1.0.1", "swagger-tools": "^0.10.4", "temp": "^0.8.3", "throng": "^4.0.0", @@ -36424,7 +36369,6 @@ "esmock": "^2.1.0", "express": "^4.18.2", "heap": "^0.2.6", - "JSONStream": "^1.3.5", "line-reader": "^0.2.4", "lodash": "^4.17.20", "mongo-uri": "^0.1.2", @@ -36438,7 +36382,6 @@ "devDependencies": { "chai": "^4.3.6", "chai-as-promised": "^7.1.1", - "memorystream": "0.3.1", "mocha": "^10.2.0", "multer": "^1.4.2", "nock": "^12.0.3", @@ -37289,7 +37232,6 @@ "body-parser": "^1.19.0", "bootstrap": "^3.4.1", "bowser": "^2.11.0", - "bufferedstream": "1.6.0", "bull": "^3.18.0", "bunyan": "^1.8.15", "cache-flow": "^1.7.4", @@ -44181,6 +44123,7 @@ "@overleaf/o-error": "*", "@overleaf/redis-wrapper": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "@pollyjs/adapter-node-http": "^4.0.3", "@pollyjs/core": "^4.0.2", "@pollyjs/persister-fs": "^4.0.2", @@ -44219,7 +44162,6 @@ "sequelize-cli": "^6.6.0", "sinon": "^9.2.4", "sinon-chai": "^3.7.0", - "stream-buffers": "^3.0.2", "underscore": "^1.9.2", "yargs": "^17.0.0" }, @@ -44478,6 +44420,7 @@ "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "async": "^3.2.2", "body-parser": "^1.19.0", "bson": "^1.1.4", @@ -44493,8 +44436,7 @@ "request": "^2.88.2", "sandboxed-module": "~2.0.4", "sinon": "~9.0.2", - "sinon-chai": "^3.7.0", - "streamifier": "^0.1.1" + "sinon-chai": "^3.7.0" }, "dependencies": { "buffer": { @@ -44643,6 +44585,7 @@ "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "aws-sdk": "^2.718.0", "body-parser": "^1.19.0", "bunyan": "^1.8.15", @@ -44659,7 +44602,6 @@ "sandboxed-module": "2.0.4", "sinon": "9.0.2", "sinon-chai": "^3.7.0", - "stream-buffers": "~0.2.6", "streamifier": "^0.1.1", "timekeeper": "^2.2.0", "tiny-async-pool": "^1.1.0" @@ -44686,11 +44628,6 @@ "supports-color": "^7.1.0" } }, - "stream-buffers": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-0.2.6.tgz", - "integrity": "sha1-GBwI1bs2kARfaUAbmuanoM8zE/w=" - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -44985,10 +44922,8 @@ "esmock": "^2.1.0", "express": "^4.18.2", "heap": "^0.2.6", - "JSONStream": "^1.3.5", "line-reader": "^0.2.4", "lodash": "^4.17.20", - "memorystream": "0.3.1", "mocha": "^10.2.0", "mongo-uri": "^0.1.2", "mongodb": "^4.11.0", @@ -45275,6 +45210,37 @@ } } }, + "@overleaf/stream-utils": { + "version": "file:libraries/stream-utils", + "requires": { + "chai": "^4.3.7", + "mocha": "*" + }, + "dependencies": { + "chai": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz", + "integrity": "sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==", + "requires": { + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^4.1.2", + "get-func-name": "^2.0.0", + "loupe": "^2.3.1", + "pathval": "^1.1.1", + "type-detect": "^4.0.5" + } + }, + "deep-eql": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz", + "integrity": "sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==", + "requires": { + "type-detect": "^4.0.0" + } + } + } + }, "@overleaf/templates": { "version": "file:services/templates", "requires": { @@ -45743,7 +45709,6 @@ "body-parser": "^1.19.0", "bootstrap": "^3.4.1", "bowser": "^2.11.0", - "bufferedstream": "1.6.0", "bull": "^3.18.0", "bunyan": "^1.8.15", "c8": "^7.2.0", @@ -49049,8 +49014,7 @@ "ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", - "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", - "dev": true + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==" }, "ansi-escape-sequences": { "version": "4.1.0", @@ -49138,7 +49102,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "devOptional": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -49382,8 +49345,7 @@ "assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", - "dev": true + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==" }, "ast-types-flow": { "version": "0.0.7", @@ -49872,8 +49834,7 @@ "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "devOptional": true + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" }, "bindings": { "version": "1.5.0", @@ -50073,8 +50034,7 @@ "browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", - "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", - "dev": true + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" }, "browserslist": { "version": "4.21.5", @@ -50145,11 +50105,6 @@ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, - "bufferedstream": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/bufferedstream/-/bufferedstream-1.6.0.tgz", - "integrity": "sha1-G1a+ZxMhYtn0aLyIbdLJFg3fKII=" - }, "buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", @@ -50644,8 +50599,7 @@ "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", - "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", - "dev": true + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=" }, "check-more-types": { "version": "2.24.0", @@ -50690,7 +50644,6 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "devOptional": true, "requires": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -52842,8 +52795,7 @@ "diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", - "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", - "dev": true + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==" }, "diff-match-patch": { "version": "git+ssh://git@github.com/overleaf/diff-match-patch.git#89805f9c671a77a263fc53461acd62aa7498f688", @@ -55265,8 +55217,7 @@ "flat": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", - "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", - "dev": true + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==" }, "flat-cache": { "version": "3.0.4", @@ -55593,7 +55544,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "optional": true }, "fstream": { @@ -55707,8 +55657,7 @@ "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", - "dev": true + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, "get-intrinsic": { "version": "1.2.0", @@ -55794,7 +55743,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "devOptional": true, "requires": { "is-glob": "^4.0.1" } @@ -56601,8 +56549,7 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" }, "heap": { "version": "0.2.7", @@ -57308,7 +57255,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "devOptional": true, "requires": { "binary-extensions": "^2.0.0" } @@ -57379,8 +57325,7 @@ "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", - "devOptional": true + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=" }, "is-fullwidth-code-point": { "version": "3.0.0", @@ -57404,7 +57349,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, "requires": { "is-extglob": "^2.1.1" } @@ -57611,8 +57555,7 @@ "is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", - "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "dev": true + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==" }, "is-utf8": { "version": "0.2.1", @@ -59715,7 +59658,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, "requires": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" @@ -59725,7 +59667,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -59734,7 +59675,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -59744,7 +59684,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "requires": { "has-flag": "^4.0.0" } @@ -59824,7 +59763,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.1.tgz", "integrity": "sha512-EN1D3jyVmaX4tnajVlfbREU4axL647hLec1h/PXAb8CPDMJiYitcWF2UeLVNttRqaIqQs4x+mRvXf+d+TlDrCA==", - "dev": true, "requires": { "get-func-name": "^2.0.0" } @@ -60487,7 +60425,6 @@ "version": "10.2.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", - "dev": true, "requires": { "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", @@ -60516,7 +60453,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "requires": { "balanced-match": "^1.0.0" } @@ -60525,7 +60461,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "requires": { "ms": "2.1.2" }, @@ -60533,8 +60468,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, @@ -60542,7 +60476,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", - "dev": true, "requires": { "brace-expansion": "^2.0.1" } @@ -60551,7 +60484,6 @@ "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dev": true, "requires": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -60884,8 +60816,7 @@ "nanoid": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", - "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==" }, "native-promise-only": { "version": "0.8.1", @@ -61724,6 +61655,7 @@ "@overleaf/metrics": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", + "@overleaf/stream-utils": "^0.1.0", "archiver": "^5.3.0", "basic-auth": "^2.0.1", "benny": "^3.7.1", @@ -61752,7 +61684,6 @@ "overleaf-editor-core": "*", "pg": "^8.7.1", "sinon": "^9.0.2", - "string-to-stream": "^1.0.1", "swagger-client": "^3.10.0", "swagger-tools": "^0.10.4", "temp": "^0.8.3", @@ -62338,8 +62269,7 @@ "pathval": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", - "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", - "dev": true + "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==" }, "pause": { "version": "0.0.1", @@ -63627,7 +63557,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "devOptional": true, "requires": { "picomatch": "^2.2.1" } @@ -64697,7 +64626,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", - "dev": true, "requires": { "randombytes": "^2.1.0" }, @@ -64706,7 +64634,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, "requires": { "safe-buffer": "^5.1.0" } @@ -65449,11 +65376,6 @@ "internal-slot": "^1.0.4" } }, - "stream-buffers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz", - "integrity": "sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==" - }, "stream-connect": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/stream-connect/-/stream-connect-1.0.2.tgz", @@ -65504,7 +65426,8 @@ "streamifier": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/streamifier/-/streamifier-0.1.1.tgz", - "integrity": "sha1-l+mNj6TRBdYqJpHR3AfoINuN/E8=" + "integrity": "sha1-l+mNj6TRBdYqJpHR3AfoINuN/E8=", + "dev": true }, "streamroller": { "version": "3.0.2", @@ -65547,44 +65470,6 @@ "resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz", "integrity": "sha512-Yptehjogou2xm4UJbxJ4CxgZx12HBfeystp0y3x7s4Dj32ltVVG1Gg8YhKjHZkHicuKpZX/ffilA8505VbUbpw==" }, - "string-to-stream": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz", - "integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==", - "requires": { - "inherits": "^2.0.1", - "readable-stream": "^2.1.0" - }, - "dependencies": { - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } - } - }, "string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -66855,8 +66740,7 @@ "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" }, "type-fest": { "version": "0.20.2", @@ -68168,7 +68052,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", - "dev": true, "requires": { "camelcase": "^6.0.0", "decamelize": "^4.0.0", @@ -68179,14 +68062,12 @@ "decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", - "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", - "dev": true + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==" }, "is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", - "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", - "dev": true + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==" } } }, diff --git a/services/docstore/app/js/DocArchiveManager.js b/services/docstore/app/js/DocArchiveManager.js index 35eeac8768..07030940cc 100644 --- a/services/docstore/app/js/DocArchiveManager.js +++ b/services/docstore/app/js/DocArchiveManager.js @@ -4,7 +4,7 @@ const Errors = require('./Errors') const logger = require('@overleaf/logger') const Settings = require('@overleaf/settings') const crypto = require('crypto') -const Streamifier = require('streamifier') +const { ReadableString } = require('@overleaf/stream-utils') const RangeManager = require('./RangeManager') const PersistorManager = require('./PersistorManager') const pMap = require('p-map') @@ -92,7 +92,7 @@ async function archiveDoc(projectId, docId) { } const md5 = crypto.createHash('md5').update(json).digest('hex') - const stream = Streamifier.createReadStream(json) + const stream = new ReadableString(json) await PersistorManager.sendStream(Settings.docstore.bucket, key, stream, { sourceMd5: md5, }) diff --git a/services/docstore/package.json b/services/docstore/package.json index 683df5e1db..aa39d55804 100644 --- a/services/docstore/package.json +++ b/services/docstore/package.json @@ -21,6 +21,7 @@ "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "async": "^3.2.2", "body-parser": "^1.19.0", "bson": "^1.1.4", @@ -30,8 +31,7 @@ "lodash": "^4.17.21", "mongodb": "^4.11.0", "p-map": "^4.0.0", - "request": "^2.88.2", - "streamifier": "^0.1.1" + "request": "^2.88.2" }, "devDependencies": { "@google-cloud/storage": "^6.10.1", diff --git a/services/docstore/test/acceptance/js/ArchiveDocsTests.js b/services/docstore/test/acceptance/js/ArchiveDocsTests.js index 5566ca2855..0345eb1a1b 100644 --- a/services/docstore/test/acceptance/js/ArchiveDocsTests.js +++ b/services/docstore/test/acceptance/js/ArchiveDocsTests.js @@ -19,10 +19,10 @@ const DocstoreApp = require('./helpers/DocstoreApp') const DocstoreClient = require('./helpers/DocstoreClient') const { Storage } = require('@google-cloud/storage') const Persistor = require('../../../app/js/PersistorManager') -const Streamifier = require('streamifier') +const { ReadableString } = require('@overleaf/stream-utils') function uploadContent(path, json, callback) { - const stream = Streamifier.createReadStream(JSON.stringify(json)) + const stream = new ReadableString(JSON.stringify(json)) Persistor.sendStream(Settings.docstore.bucket, path, stream) .then(() => callback()) .catch(callback) diff --git a/services/docstore/test/unit/js/DocArchiveManagerTests.js b/services/docstore/test/unit/js/DocArchiveManagerTests.js index 4892392b20..cfa0cc34f1 100644 --- a/services/docstore/test/unit/js/DocArchiveManagerTests.js +++ b/services/docstore/test/unit/js/DocArchiveManagerTests.js @@ -12,7 +12,7 @@ describe('DocArchiveManager', function () { RangeManager, Settings, Crypto, - Streamifier, + StreamUtils, HashDigest, HashUpdate, archivedDocs, @@ -42,8 +42,8 @@ describe('DocArchiveManager', function () { Crypto = { createHash: sinon.stub().returns({ update: HashUpdate }), } - Streamifier = { - createReadStream: sinon.stub().returns({ stream: 'readStream' }), + StreamUtils = { + ReadableString: sinon.stub().returns({ stream: 'readStream' }), } projectId = ObjectId() @@ -158,7 +158,7 @@ describe('DocArchiveManager', function () { requires: { '@overleaf/settings': Settings, crypto: Crypto, - streamifier: Streamifier, + '@overleaf/stream-utils': StreamUtils, './MongoManager': MongoManager, './RangeManager': RangeManager, './PersistorManager': PersistorManager, @@ -185,7 +185,7 @@ describe('DocArchiveManager', function () { it('should add the schema version', async function () { await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[1]._id) - expect(Streamifier.createReadStream).to.have.been.calledWith( + expect(StreamUtils.ReadableString).to.have.been.calledWith( sinon.match(/"schema_v":1/) ) }) @@ -219,7 +219,7 @@ describe('DocArchiveManager', function () { it('should create a stream from the encoded json and send it', async function () { await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id) - expect(Streamifier.createReadStream).to.have.been.calledWith( + expect(StreamUtils.ReadableString).to.have.been.calledWith( archivedDocJson ) expect(PersistorManager.sendStream).to.have.been.calledWith( diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js index 512dca8e32..fb296513b5 100644 --- a/services/filestore/app/js/HealthCheckController.js +++ b/services/filestore/app/js/HealthCheckController.js @@ -1,7 +1,7 @@ const fs = require('fs') const path = require('path') const Settings = require('@overleaf/settings') -const streamBuffers = require('stream-buffers') +const { WritableBuffer } = require('@overleaf/stream-utils') const { promisify } = require('util') const Stream = require('stream') @@ -23,9 +23,7 @@ async function checkCanGetFiles() { const key = `${projectId}/${fileId}` const bucket = Settings.filestore.stores.user_files - const buffer = new streamBuffers.WritableStreamBuffer({ - initialSize: 100, - }) + const buffer = new WritableBuffer({ initialSize: 100 }) const sourceStream = await FileHandler.getFile(bucket, key, {}) try { diff --git a/services/filestore/package.json b/services/filestore/package.json index 7db113f801..9386d8d263 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -23,6 +23,7 @@ "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", "@overleaf/settings": "*", + "@overleaf/stream-utils": "^0.1.0", "body-parser": "^1.19.0", "bunyan": "^1.8.15", "express": "^4.18.2", @@ -30,7 +31,6 @@ "lodash.once": "^4.1.1", "node-fetch": "^2.6.7", "range-parser": "^1.2.1", - "stream-buffers": "~0.2.6", "tiny-async-pool": "^1.1.0" }, "devDependencies": { diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js index 4d3d8e6a00..2e1dd564a6 100644 --- a/services/filestore/test/acceptance/js/FilestoreTests.js +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -137,13 +137,6 @@ describe('Filestore', function () { expect(body).to.contain('up') }) - it('should send a 200 for the health-check endpoint', async function () { - const response = await fetch(`${filestoreUrl}/health_check`) - expect(response.status).to.equal(200) - const body = await response.text() - expect(body).to.equal('OK') - }) - describe('with a file on the server', function () { let fileId, fileUrl, constantFileContent @@ -198,6 +191,17 @@ describe('Filestore', function () { expect(body).to.equal(constantFileContent) }) + it('should send a 200 for the health-check endpoint using the file', async function () { + Settings.health_check = { + project_id: projectId, + file_id: fileId, + } + const response = await fetch(`${filestoreUrl}/health_check`) + expect(response.status).to.equal(200) + const body = await response.text() + expect(body).to.equal('OK') + }) + it('should not leak a socket', async function () { await fetch(fileUrl) await expectNoSockets() diff --git a/services/history-v1/package.json b/services/history-v1/package.json index f0307756e6..623a95e5c6 100644 --- a/services/history-v1/package.json +++ b/services/history-v1/package.json @@ -10,6 +10,7 @@ "@overleaf/metrics": "*", "@overleaf/o-error": "*", "@overleaf/object-persistor": "*", + "@overleaf/stream-utils": "^0.1.0", "archiver": "^5.3.0", "basic-auth": "^2.0.1", "bluebird": "^3.7.2", @@ -31,7 +32,6 @@ "mongodb": "^4.11.0", "overleaf-editor-core": "*", "pg": "^8.7.1", - "string-to-stream": "^1.0.1", "swagger-tools": "^0.10.4", "temp": "^0.8.3", "throng": "^4.0.0", diff --git a/services/history-v1/storage/lib/blob_store/index.js b/services/history-v1/storage/lib/blob_store/index.js index ae286389b3..8a06ebcf2d 100644 --- a/services/history-v1/storage/lib/blob_store/index.js +++ b/services/history-v1/storage/lib/blob_store/index.js @@ -3,7 +3,7 @@ const config = require('config') const fs = require('fs') const isValidUtf8 = require('utf-8-validate') -const stringToStream = require('string-to-stream') +const { ReadableString } = require('@overleaf/stream-utils') const core = require('overleaf-editor-core') const objectPersistor = require('@overleaf/object-persistor') @@ -162,9 +162,9 @@ class BlobStore { return existingBlob } const newBlob = new Blob(hash, Buffer.byteLength(string), string.length) - // Note: the stringToStream is to work around a bug in the AWS SDK: it won't + // Note: the ReadableString is to work around a bug in the AWS SDK: it won't // allow Body to be blank. - await uploadBlob(this.projectId, newBlob, stringToStream(string)) + await uploadBlob(this.projectId, newBlob, new ReadableString(string)) await this.backend.insertBlob(this.projectId, newBlob) return newBlob } diff --git a/services/history-v1/storage/lib/streams.js b/services/history-v1/storage/lib/streams.js index 76c553c27f..21f360efb5 100644 --- a/services/history-v1/storage/lib/streams.js +++ b/services/history-v1/storage/lib/streams.js @@ -7,8 +7,8 @@ const BPromise = require('bluebird') const zlib = require('zlib') -const stringToStream = require('string-to-stream') -const { pipeline, Writable } = require('stream') +const { WritableBuffer, ReadableString } = require('@overleaf/stream-utils') +const { pipeline } = require('stream') function promisePipe(readStream, writeStream) { return new BPromise(function (resolve, reject) { @@ -33,26 +33,6 @@ function promisePipe(readStream, writeStream) { */ exports.promisePipe = promisePipe -class WritableBuffer extends Writable { - constructor(options) { - super(options) - this.buffers = [] - } - - _write(chunk, encoding, callback) { - this.buffers.push(chunk) - callback() - } - - _final(callback) { - callback() - } - - contents() { - return Buffer.concat(this.buffers) - } -} - function readStreamToBuffer(readStream) { return new BPromise(function (resolve, reject) { const bufferStream = new WritableBuffer() @@ -100,7 +80,7 @@ exports.gunzipStreamToBuffer = gunzipStreamToBuffer function gzipStringToStream(string) { const gzip = zlib.createGzip() - return stringToStream(string).pipe(gzip) + return new ReadableString(string).pipe(gzip) } /** diff --git a/services/project-history/package.json b/services/project-history/package.json index aba530479a..39d0b80b51 100644 --- a/services/project-history/package.json +++ b/services/project-history/package.json @@ -34,7 +34,6 @@ "esmock": "^2.1.0", "express": "^4.18.2", "heap": "^0.2.6", - "JSONStream": "^1.3.5", "line-reader": "^0.2.4", "lodash": "^4.17.20", "mongo-uri": "^0.1.2", @@ -48,7 +47,6 @@ "devDependencies": { "chai": "^4.3.6", "chai-as-promised": "^7.1.1", - "memorystream": "0.3.1", "mocha": "^10.2.0", "multer": "^1.4.2", "nock": "^12.0.3", diff --git a/services/web/package.json b/services/web/package.json index 073d5ab7d7..aa78729993 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -137,7 +137,6 @@ "body-parser": "^1.19.0", "bootstrap": "^3.4.1", "bowser": "^2.11.0", - "bufferedstream": "1.6.0", "bull": "^3.18.0", "bunyan": "^1.8.15", "cache-flow": "^1.7.4", diff --git a/services/web/test/unit/src/ThirdPartyDataStore/UpdateMergerTests.js b/services/web/test/unit/src/ThirdPartyDataStore/UpdateMergerTests.js index 302a6f6be2..4cda226117 100644 --- a/services/web/test/unit/src/ThirdPartyDataStore/UpdateMergerTests.js +++ b/services/web/test/unit/src/ThirdPartyDataStore/UpdateMergerTests.js @@ -1,7 +1,7 @@ const SandboxedModule = require('sandboxed-module') const sinon = require('sinon') const { expect } = require('chai') -const BufferedStream = require('bufferedstream') +const { Writable } = require('stream') const { ObjectId } = require('mongodb') const MODULE_PATH = @@ -32,7 +32,7 @@ describe('UpdateMerger :', function () { \\date{June 2011}` this.docLines = this.fileContents.split('\n') this.source = 'dropbox' - this.updateRequest = new BufferedStream() + this.updateRequest = new Writable() this.fsPromises = { unlink: sinon.stub().resolves(),