diff --git a/libraries/fetch-utils/.dockerignore b/libraries/fetch-utils/.dockerignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/libraries/fetch-utils/.dockerignore @@ -0,0 +1 @@ +node_modules/ diff --git a/libraries/fetch-utils/.gitignore b/libraries/fetch-utils/.gitignore new file mode 100644 index 0000000000..edb0f85350 --- /dev/null +++ b/libraries/fetch-utils/.gitignore @@ -0,0 +1,3 @@ + +# managed by monorepo$ bin/update_build_scripts +.npmrc diff --git a/libraries/fetch-utils/.mocharc.json b/libraries/fetch-utils/.mocharc.json new file mode 100644 index 0000000000..c492858ff0 --- /dev/null +++ b/libraries/fetch-utils/.mocharc.json @@ -0,0 +1,6 @@ +{ + "ui": "bdd", + "recursive": "true", + "reporter": "spec", + "require": "test/setup.js" +} diff --git a/libraries/fetch-utils/.nvmrc b/libraries/fetch-utils/.nvmrc new file mode 100644 index 0000000000..3876fd4986 --- /dev/null +++ b/libraries/fetch-utils/.nvmrc @@ -0,0 +1 @@ +18.16.1 diff --git a/libraries/fetch-utils/Dockerfile b/libraries/fetch-utils/Dockerfile new file mode 100644 index 0000000000..e515c7f23b --- /dev/null +++ b/libraries/fetch-utils/Dockerfile @@ -0,0 +1,5 @@ +FROM node:18.16.1 + +WORKDIR /app + +USER node diff --git a/libraries/fetch-utils/buildscript.txt b/libraries/fetch-utils/buildscript.txt new file mode 100644 index 0000000000..8aedd71c00 --- /dev/null +++ b/libraries/fetch-utils/buildscript.txt @@ -0,0 +1,11 @@ +fetch-utils +--dependencies=None +--docker-repos=gcr.io/overleaf-ops +--env-add= +--env-pass-through= +--esmock-loader=False +--has-custom-nodemon=True +--is-library=True +--node-version=18.16.1 +--public-repo=False +--script-version=4.3.0 diff --git a/libraries/fetch-utils/index.js b/libraries/fetch-utils/index.js new file mode 100644 index 0000000000..5f30ac25b3 --- /dev/null +++ b/libraries/fetch-utils/index.js @@ -0,0 +1,228 @@ +const _ = require('lodash') +const { Readable } = require('stream') +const OError = require('@overleaf/o-error') +const fetch = require('node-fetch') + +/** + * Make a request and return the parsed JSON response. + * + * @param {string | URL} url - request URL + * @param {object} opts - fetch options + * @return {Promise} the parsed JSON response + * @throws {RequestFailedError} if the response has a failure status code + */ +async function fetchJson(url, opts = {}) { + const { json } = await fetchJsonWithResponse(url, opts) + return json +} + +async function fetchJsonWithResponse(url, opts = {}) { + const { fetchOpts } = parseOpts(opts) + fetchOpts.headers = fetchOpts.headers ?? {} + fetchOpts.headers.Accept = 'application/json' + + const response = await performRequest(url, fetchOpts) + if (!response.ok) { + const body = await maybeGetResponseBody(response) + throw new RequestFailedError(url, opts, response, body) + } + + const json = await response.json() + return { json, response } +} + +/** + * Make a request and return a stream. + * + * If the response body is destroyed, the request is aborted. + * + * @param {string | URL} url - request URL + * @param {object} opts - fetch options + * @return {Promise} + * @throws {RequestFailedError} if the response has a failure status code + */ +async function fetchStream(url, opts = {}) { + const { stream } = await fetchStreamWithResponse(url, opts) + return stream +} + +async function fetchStreamWithResponse(url, opts = {}) { + const { fetchOpts, abortController } = parseOpts(opts) + const response = await performRequest(url, fetchOpts) + + if (!response.ok) { + const body = await maybeGetResponseBody(response) + throw new RequestFailedError(url, opts, response, body) + } + + abortOnDestroyedResponse(abortController, response) + + const stream = response.body + return { stream, response } +} + +/** + * Make a request and discard the response. + * + * @param {string | URL} url - request URL + * @param {object} opts - fetch options + * @return {Promise} + * @throws {RequestFailedError} if the response has a failure status code + */ +async function fetchNothing(url, opts = {}) { + const { fetchOpts } = parseOpts(opts) + const response = await performRequest(url, fetchOpts) + if (!response.ok) { + const body = await maybeGetResponseBody(response) + throw new RequestFailedError(url, opts, response, body) + } + await discardResponseBody(response) + return response +} + +/** + * Make a request and return a string. + * + * @param {string | URL} url - request URL + * @param {object} opts - fetch options + * @return {Promise} + * @throws {RequestFailedError} if the response has a failure status code + */ +async function fetchString(url, opts = {}) { + const { body } = await fetchStringWithResponse(url, opts) + return body +} + +async function fetchStringWithResponse(url, opts = {}) { + const { fetchOpts } = parseOpts(opts) + const response = await performRequest(url, fetchOpts) + if (!response.ok) { + const body = await maybeGetResponseBody(response) + throw new RequestFailedError(url, opts, response, body) + } + const body = await response.text() + return { body, response } +} + +class RequestFailedError extends OError { + constructor(url, opts, response, body) { + super('request failed', { + url, + method: opts.method ?? 'GET', + status: response.status, + }) + + this.response = response + if (body != null) { + this.body = body + } + } +} + +function parseOpts(opts) { + const fetchOpts = _.omit(opts, ['json', 'signal', 'basicAuth']) + if (opts.json) { + setupJsonBody(fetchOpts, opts.json) + } + if (opts.basicAuth) { + setupBasicAuth(fetchOpts, opts.basicAuth) + } + + const abortController = new AbortController() + fetchOpts.signal = abortController.signal + if (opts.signal) { + abortOnSignal(abortController, opts.signal) + } + if (opts.body instanceof Readable) { + abortOnDestroyedRequest(abortController, fetchOpts.body) + } + return { fetchOpts, abortController } +} + +function setupJsonBody(fetchOpts, json) { + fetchOpts.body = JSON.stringify(json) + fetchOpts.headers = fetchOpts.headers ?? {} + fetchOpts.headers['Content-Type'] = 'application/json' +} + +function setupBasicAuth(fetchOpts, basicAuth) { + fetchOpts.headers = fetchOpts.headers ?? {} + fetchOpts.headers.Authorization = + 'Basic ' + + Buffer.from(`${basicAuth.user}:${basicAuth.password}`).toString('base64') +} + +function abortOnSignal(abortController, signal) { + const listener = () => { + abortController.abort(signal.reason) + } + if (signal.aborted) { + abortController.abort(signal.reason) + } + signal.addEventListener('abort', listener) +} + +function abortOnDestroyedRequest(abortController, stream) { + stream.on('close', () => { + if (!stream.readableEnded) { + abortController.abort() + } + }) +} + +function abortOnDestroyedResponse(abortController, response) { + response.body.on('close', () => { + if (!response.bodyUsed) { + abortController.abort() + } + }) +} + +async function performRequest(url, fetchOpts) { + let response + try { + response = await fetch(url, fetchOpts) + } catch (err) { + if (fetchOpts.body instanceof Readable) { + fetchOpts.body.destroy() + } + throw OError.tag(err, err.message, { + url, + method: fetchOpts.method ?? 'GET', + }) + } + if (fetchOpts.body instanceof Readable) { + response.body.on('close', () => { + if (!fetchOpts.body.readableEnded) { + fetchOpts.body.destroy() + } + }) + } + return response +} + +async function discardResponseBody(response) { + // eslint-disable-next-line no-unused-vars + for await (const chunk of response.body) { + // discard the body + } +} + +async function maybeGetResponseBody(response) { + try { + return await response.text() + } catch (err) { + return null + } +} + +module.exports = { + fetchJson, + fetchJsonWithResponse, + fetchStream, + fetchStreamWithResponse, + fetchNothing, + fetchString, + fetchStringWithResponse, + RequestFailedError, +} diff --git a/libraries/fetch-utils/package.json b/libraries/fetch-utils/package.json new file mode 100644 index 0000000000..09f1daaa60 --- /dev/null +++ b/libraries/fetch-utils/package.json @@ -0,0 +1,29 @@ +{ + "name": "@overleaf/fetch-utils", + "version": "0.1.0", + "description": "utilities for node-fetch", + "main": "index.js", + "scripts": { + "test": "npm run lint && npm run format && npm run test:unit", + "test:unit": "mocha", + "lint": "eslint --max-warnings 0 --format unix .", + "lint:fix": "eslint --fix .", + "format": "prettier --list-different $PWD/'**/*.js'", + "format:fix": "prettier --write $PWD/'**/*.js'", + "test:ci": "npm run test:unit" + }, + "author": "Overleaf (https://www.overleaf.com)", + "license": "AGPL-3.0-only", + "devDependencies": { + "body-parser": "^1.20.2", + "chai": "^4.3.6", + "chai-as-promised": "^7.1.1", + "express": "^4.18.2", + "mocha": "^10.2.0" + }, + "dependencies": { + "@overleaf/o-error": "*", + "lodash": "^4.17.21", + "node-fetch": "^2.6.11" + } +} diff --git a/libraries/fetch-utils/test/setup.js b/libraries/fetch-utils/test/setup.js new file mode 100644 index 0000000000..09068185a8 --- /dev/null +++ b/libraries/fetch-utils/test/setup.js @@ -0,0 +1,4 @@ +const chai = require('chai') +const chaiAsPromised = require('chai-as-promised') + +chai.use(chaiAsPromised) diff --git a/libraries/fetch-utils/test/unit/FetchUtilsTests.js b/libraries/fetch-utils/test/unit/FetchUtilsTests.js new file mode 100644 index 0000000000..af3d1f8df7 --- /dev/null +++ b/libraries/fetch-utils/test/unit/FetchUtilsTests.js @@ -0,0 +1,230 @@ +const { expect } = require('chai') +const { FetchError, AbortError } = require('node-fetch') +const { Readable } = require('stream') +const { once } = require('events') +const { TestServer } = require('./helpers/TestServer') +const { + fetchJson, + fetchStream, + fetchNothing, + fetchString, + RequestFailedError, +} = require('../..') + +const PORT = 30001 + +describe('fetch-utils', function () { + before(async function () { + this.server = new TestServer() + await this.server.start(PORT) + this.url = path => `http://localhost:${PORT}${path}` + }) + + after(async function () { + await this.server.stop() + }) + + describe('fetchJson', function () { + it('parses a JSON response', async function () { + const json = await fetchJson(this.url('/json/hello')) + expect(json).to.deep.equal({ msg: 'hello' }) + }) + + it('parses JSON in the request', async function () { + const json = await fetchJson(this.url('/json/add'), { + method: 'POST', + json: { a: 2, b: 3 }, + }) + expect(json).to.deep.equal({ sum: 5 }) + }) + + it('accepts stringified JSON as body', async function () { + const json = await fetchJson(this.url('/json/add'), { + method: 'POST', + body: JSON.stringify({ a: 2, b: 3 }), + headers: { 'Content-Type': 'application/json' }, + }) + expect(json).to.deep.equal({ sum: 5 }) + }) + + it('throws a FetchError when the payload is not JSON', async function () { + await expect(fetchJson(this.url('/hello'))).to.be.rejectedWith(FetchError) + }) + + it('aborts the request if JSON parsing fails', async function () { + await expect(fetchJson(this.url('/large'))).to.be.rejectedWith(FetchError) + await expectRequestAborted(this.server.lastReq) + }) + + it('handles errors when the payload is JSON', async function () { + await expect(fetchJson(this.url('/json/500'))).to.be.rejectedWith( + RequestFailedError + ) + await expectRequestAborted(this.server.lastReq) + }) + + it('handles errors when the payload is not JSON', async function () { + await expect(fetchJson(this.url('/500'))).to.be.rejectedWith( + RequestFailedError + ) + await expectRequestAborted(this.server.lastReq) + }) + + it('supports abort signals', async function () { + await expect( + fetchJson(this.url('/hang'), { signal: AbortSignal.timeout(10) }) + ).to.be.rejectedWith(AbortError) + await expectRequestAborted(this.server.lastReq) + }) + + it('supports basic auth', async function () { + const json = await fetchJson(this.url('/json/basic-auth'), { + basicAuth: { user: 'user', password: 'pass' }, + }) + expect(json).to.deep.equal({ key: 'verysecret' }) + }) + + it("destroys the request body if it doesn't get consumed", async function () { + const stream = Readable.from(infiniteIterator()) + await fetchJson(this.url('/json/ignore-request'), { + method: 'POST', + body: stream, + }) + expect(stream.destroyed).to.be.true + }) + }) + + describe('fetchStream', function () { + it('returns a stream', async function () { + const stream = await fetchStream(this.url('/large')) + const text = await streamToString(stream) + expect(text).to.equal(this.server.largePayload) + }) + + it('aborts the request when the stream is destroyed', async function () { + const stream = await fetchStream(this.url('/large')) + stream.destroy() + await expectRequestAborted(this.server.lastReq) + }) + + it('aborts the request when the request body is destroyed', async function () { + const stream = Readable.from(infiniteIterator()) + const promise = fetchStream(this.url('/hang'), { + method: 'POST', + body: stream, + }) + stream.destroy() + await expect(promise).to.be.rejectedWith(AbortError) + await expectRequestAborted(this.server.lastReq) + }) + + it('handles errors', async function () { + await expect(fetchStream(this.url('/500'))).to.be.rejectedWith( + RequestFailedError + ) + await expectRequestAborted(this.server.lastReq) + }) + + it('supports abort signals', async function () { + await expect( + fetchStream(this.url('/hang'), { signal: AbortSignal.timeout(10) }) + ).to.be.rejectedWith(AbortError) + await expectRequestAborted(this.server.lastReq) + }) + + it('destroys the request body when an error occurs', async function () { + const stream = Readable.from(infiniteIterator()) + await expect( + fetchStream(this.url('/hang'), { + body: stream, + signal: AbortSignal.timeout(10), + }) + ).to.be.rejectedWith(AbortError) + expect(stream.destroyed).to.be.true + }) + }) + + describe('fetchNothing', function () { + it('closes the connection', async function () { + await fetchNothing(this.url('/large')) + await expectRequestAborted(this.server.lastReq) + }) + + it('aborts the request when the request body is destroyed', async function () { + const stream = Readable.from(infiniteIterator()) + const promise = fetchNothing(this.url('/hang'), { + method: 'POST', + body: stream, + }) + stream.destroy() + await expect(promise).to.be.rejectedWith(AbortError) + await expectRequestAborted(this.server.lastReq) + }) + + it("doesn't abort the request if the request body ends normally", async function () { + const stream = Readable.from('hello there') + await fetchNothing(this.url('/sink'), { method: 'POST', body: stream }) + }) + + it('handles errors', async function () { + await expect(fetchNothing(this.url('/500'))).to.be.rejectedWith( + RequestFailedError + ) + await expectRequestAborted(this.server.lastReq) + }) + + it('supports abort signals', async function () { + await expect( + fetchNothing(this.url('/hang'), { signal: AbortSignal.timeout(10) }) + ).to.be.rejectedWith(AbortError) + await expectRequestAborted(this.server.lastReq) + }) + + it('destroys the request body when an error occurs', async function () { + const stream = Readable.from(infiniteIterator()) + await expect( + fetchNothing(this.url('/hang'), { + body: stream, + signal: AbortSignal.timeout(10), + }) + ).to.be.rejectedWith(AbortError) + expect(stream.destroyed).to.be.true + }) + }) + + describe('fetchString', function () { + it('returns a string', async function () { + const body = await fetchString(this.url('/hello')) + expect(body).to.equal('hello') + }) + + it('handles errors', async function () { + await expect(fetchJson(this.url('/500'))).to.be.rejectedWith( + RequestFailedError + ) + await expectRequestAborted(this.server.lastReq) + }) + }) +}) + +async function streamToString(stream) { + let s = '' + for await (const chunk of stream) { + s += chunk + } + return s +} + +async function* infiniteIterator() { + let i = 1 + while (true) { + yield `chunk ${i++}\n` + } +} + +async function expectRequestAborted(req) { + if (!req.destroyed) { + await once(req, 'close') + expect(req.destroyed).to.be.true + } +} diff --git a/libraries/fetch-utils/test/unit/helpers/TestServer.js b/libraries/fetch-utils/test/unit/helpers/TestServer.js new file mode 100644 index 0000000000..5d080445e0 --- /dev/null +++ b/libraries/fetch-utils/test/unit/helpers/TestServer.js @@ -0,0 +1,102 @@ +const express = require('express') +const bodyParser = require('body-parser') + +class TestServer { + constructor(port) { + this.app = express() + + this.app.use(bodyParser.json()) + this.app.use((req, res, next) => { + this.lastReq = req + next() + }) + + // Plain text endpoints + + this.app.get('/hello', (req, res) => { + res.send('hello') + }) + + this.largePayload = 'x'.repeat(16 * 1024 * 1024) + this.app.get('/large', (req, res) => { + res.send(this.largePayload) + }) + + this.app.get('/204', (req, res) => { + res.status(204).end() + }) + + this.app.get('/empty', (req, res) => { + res.end() + }) + + this.app.get('/500', (req, res) => { + res.sendStatus(500) + }) + + this.app.post('/sink', (req, res) => { + req.on('data', () => {}) + req.on('end', () => { + res.status(204).end() + }) + }) + + // JSON endpoints + + this.app.get('/json/hello', (req, res) => { + res.json({ msg: 'hello' }) + }) + + this.app.post('/json/add', (req, res) => { + const { a, b } = req.body + res.json({ sum: a + b }) + }) + + this.app.get('/json/500', (req, res) => { + res.status(500).json({ error: 'Internal server error' }) + }) + + this.app.get('/json/basic-auth', (req, res) => { + const expectedAuth = 'Basic ' + Buffer.from('user:pass').toString('base64') + if (req.headers.authorization === expectedAuth) { + res.json({ key: 'verysecret' }) + } else { + res.status(401).json({ error: 'unauthorized' }) + } + }) + + this.app.post('/json/ignore-request', (req, res) => { + res.json({ msg: 'hello' }) + }) + + // Never returns + + this.app.post('/hang', (req, res) => {}) + } + + start(port) { + return new Promise((resolve, reject) => { + this.server = this.app.listen(port, err => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) + } + + stop() { + return new Promise((resolve, reject) => { + this.server.close(err => { + if (err) { + reject(err) + } else { + resolve() + } + }) + }) + } +} + +module.exports = { TestServer }