overleaf/services/clsi/app/coffee/UrlFetcher.js

/*
 * decaffeinate suggestions:
 * DS102: Remove unnecessary code created because of implicit returns
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
let UrlFetcher;
const request = require("request").defaults({jar: false});
const fs = require("fs");
const logger = require("logger-sharelatex");
const settings = require("settings-sharelatex");
const URL = require('url');

const oneMinute = 60 * 1000;

module.exports = (UrlFetcher = {
	pipeUrlToFile(url, filePath, _callback) {
		if (_callback == null) { _callback = function(error) {}; }
		const callbackOnce = function(error) {
			if (timeoutHandler != null) { clearTimeout(timeoutHandler); }
			_callback(error);
			return _callback = function() {};
		};

		if (settings.filestoreDomainOveride != null) {
			const p = URL.parse(url).path;
			url = `${settings.filestoreDomainOveride}${p}`;
		}
		var timeoutHandler = setTimeout(function() {
			timeoutHandler = null;
			logger.error({url, filePath}, "Timed out downloading file to cache");
			return callbackOnce(new Error(`Timed out downloading file to cache ${url}`));
		}
			// FIXME: maybe need to close fileStream here
		, 3 * oneMinute);

		logger.log({url, filePath}, "started downloading url to cache");
		const urlStream = request.get({url, timeout: oneMinute});
		urlStream.pause(); // stop data flowing until we are ready

		// attach handlers before setting up pipes
		urlStream.on("error", function(error) {
			logger.error({err: error, url, filePath}, "error downloading url");
			return callbackOnce(error || new Error(`Something went wrong downloading the URL ${url}`));
		});

		urlStream.on("end", () => logger.log({url, filePath}, "finished downloading file into cache"));

		return urlStream.on("response", function(res) {
			if ((res.statusCode >= 200) && (res.statusCode < 300)) {
				const fileStream = fs.createWriteStream(filePath);

				// attach handlers before setting up pipes
				fileStream.on('error', function(error) {
					logger.error({err: error, url, filePath}, "error writing file into cache");
					return fs.unlink(filePath, function(err) {
						if (err != null) {
							logger.err({err, filePath}, "error deleting file from cache");
						}
						return callbackOnce(error);
					});
				});

				fileStream.on('finish', function() {
					logger.log({url, filePath}, "finished writing file into cache");
					return callbackOnce();
				});

				fileStream.on('pipe', () => logger.log({url, filePath}, "piping into filestream"));

				urlStream.pipe(fileStream);
				return urlStream.resume(); // now we are ready to handle the data
			} else {
				logger.error({statusCode: res.statusCode, url, filePath}, "unexpected status code downloading url to cache");
				// https://nodejs.org/api/http.html#http_class_http_clientrequest
				// If you add a 'response' event handler, then you must consume
				// the data from the response object, either by calling
				// response.read() whenever there is a 'readable' event, or by
				// adding a 'data' handler, or by calling the .resume()
				// method. Until the data is consumed, the 'end' event will not
				// fire. Also, until the data is read it will consume memory
				// that can eventually lead to a 'process out of memory' error.
				urlStream.resume(); // discard the data
				return callbackOnce(new Error(`URL returned non-success status code: ${res.statusCode} ${url}`));
			}
		});
	}
});