Merge pull request #4819 from overleaf/jpa-pdf-caching-new-event-loop

[clsi] pdf-caching: move cpu intensive work onto a new event loop GitOrigin-RevId: 4cb5cd4528fa1c5df6a8e91f9caa38cb64d94463
2024-11-07 20:31:06 -05:00 · 2021-08-24 11:22:30 +02:00 · 2021-08-24 11:22:30 +02:00 · 56a3b0dcde
commit 56a3b0dcde
parent ac3f6114d8
11 changed files with 479 additions and 17 deletions
--- a/services/clsi/app/js/ContentCacheManager.js
+++ b/services/clsi/app/js/ContentCacheManager.js
@ -10,7 +10,36 @@ const Settings = require('@overleaf/settings')
 const OError = require('@overleaf/o-error')
 const pLimit = require('p-limit')
 const { parseXrefTable } = require('../lib/pdfjs/parseXrefTable')
-const { TimedOutError } = require('./Errors')
+const { QueueLimitReachedError, TimedOutError } = require('./Errors')
+const workerpool = require('workerpool')
+const Metrics = require('@overleaf/metrics')
+
+let WORKER_POOL
+// NOTE: Check for main thread to avoid recursive start of pool.
+if (Settings.pdfCachingEnableWorkerPool && workerpool.isMainThread) {
+  WORKER_POOL = workerpool.pool(Path.join(__dirname, 'ContentCacheWorker.js'), {
+    // Cap number of worker threads.
+    maxWorkers: Settings.pdfCachingWorkerPoolSize,
+    // Warmup workers.
+    minWorkers: Settings.pdfCachingWorkerPoolSize,
+    // Limit queue back-log
+    maxQueueSize: Settings.pdfCachingWorkerPoolBackLogLimit,
+  })
+  setInterval(() => {
+    const {
+      totalWorkers,
+      busyWorkers,
+      idleWorkers,
+      pendingTasks,
+      activeTasks,
+    } = WORKER_POOL.stats()
+    Metrics.gauge('pdf_caching_total_workers', totalWorkers)
+    Metrics.gauge('pdf_caching_busy_workers', busyWorkers)
+    Metrics.gauge('pdf_caching_idle_workers', idleWorkers)
+    Metrics.gauge('pdf_caching_pending_tasks', pendingTasks)
+    Metrics.gauge('pdf_caching_active_tasks', activeTasks)
+  }, 15 * 1000)
+}

 /**
 *
@ -20,7 +49,68 @@ const { TimedOutError } = require('./Errors')
 * @param {number} compileTime
 */
 async function update(contentDir, filePath, size, compileTime) {
+  if (Settings.pdfCachingEnableWorkerPool) {
+    return await updateOtherEventLoop(contentDir, filePath, size, compileTime)
+  } else {
+    return await updateSameEventLoop(contentDir, filePath, size, compileTime)
+  }
+}
+
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ * @param {number} compileTime
+ */
+async function updateOtherEventLoop(contentDir, filePath, size, compileTime) {
+  const timeout = getMaxOverhead(compileTime)
+  try {
+    return await WORKER_POOL.exec('doUpdateInternalNoDeadline', [
+      contentDir,
+      filePath,
+      size,
+    ]).timeout(timeout)
+  } catch (e) {
+    if (e instanceof workerpool.Promise.TimeoutError) {
+      throw new TimedOutError('context-lost-in-worker')
+    }
+    if (e.message.includes('Max queue size of ')) {
+      throw new QueueLimitReachedError()
+    }
+    throw e
+  }
+}
+
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ * @param {number} compileTime
+ */
+async function updateSameEventLoop(contentDir, filePath, size, compileTime) {
  const checkDeadline = getDeadlineChecker(compileTime)
+  return doUpdateInternal(contentDir, filePath, size, checkDeadline)
+}
+
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ */
+async function doUpdateInternalNoDeadline(contentDir, filePath, size) {
+  return doUpdateInternal(contentDir, filePath, size, () => {})
+}
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ * @param {function} checkDeadline
+ */
+async function doUpdateInternal(contentDir, filePath, size, checkDeadline) {
  const ranges = []
  const newRanges = []
  // keep track of hashes expire old ones when they reach a generation > N.
@ -233,14 +323,18 @@ async function writePdfStream(dir, hash, buffer) {
  }
 }

-function getDeadlineChecker(compileTime) {
-  const maxOverhead = Math.min(
+function getMaxOverhead(compileTime) {
+  return Math.min(
    // Adding 10s to a  40s compile time is OK.
    // Adding  1s to a   3s compile time is OK.
    Math.max(compileTime / 4, 1000),
    // Adding 30s to a 120s compile time is not OK, limit to 10s.
    Settings.pdfCachingMaxProcessingTime
  )
+}
+
+function getDeadlineChecker(compileTime) {
+  const maxOverhead = getMaxOverhead(compileTime)

  const deadline = Date.now() + maxOverhead
  let lastStage = { stage: 'start', now: Date.now() }
@ -269,5 +363,6 @@ module.exports = {
  update: callbackify(update),
  promises: {
    update,
+    doUpdateInternalNoDeadline,
  },
 }
--- a/services/clsi/app/js/ContentCacheMetrics.js
+++ b/services/clsi/app/js/ContentCacheMetrics.js
@ -22,6 +22,9 @@ function emitPdfStats(stats, timings) {
  if (stats['pdf-caching-timed-out']) {
    Metrics.inc('pdf-caching-timed-out')
  }
+  if (stats['pdf-caching-queue-limit-reached']) {
+    Metrics.inc('pdf-caching-queue-limit-reached')
+  }
  if (timings['compute-pdf-caching']) {
    emitPdfCachingStats(stats, timings)
  } else {
--- a/services/clsi/app/js/ContentCacheWorker.js
+++ b/services/clsi/app/js/ContentCacheWorker.js
@ -0,0 +1,4 @@
+const workerpool = require('workerpool')
+const ContentCacheManager = require('./ContentCacheManager')
+
+workerpool.worker(ContentCacheManager.promises)
--- a/services/clsi/app/js/Errors.js
+++ b/services/clsi/app/js/Errors.js
@ -31,9 +31,11 @@ var AlreadyCompilingError = function (message) {
 }
 AlreadyCompilingError.prototype.__proto__ = Error.prototype

+class QueueLimitReachedError extends OError {}
 class TimedOutError extends OError {}

 module.exports = Errors = {
+  QueueLimitReachedError,
  TimedOutError,
  NotFoundError,
  FilesOutOfSyncError,
--- a/services/clsi/app/js/OutputCacheManager.js
+++ b/services/clsi/app/js/OutputCacheManager.js
@ -26,7 +26,7 @@ const Metrics = require('./Metrics')

 const OutputFileOptimiser = require('./OutputFileOptimiser')
 const ContentCacheManager = require('./ContentCacheManager')
-const { TimedOutError } = require('./Errors')
+const { QueueLimitReachedError, TimedOutError } = require('./Errors')

 module.exports = OutputCacheManager = {
  CONTENT_SUBDIR: 'content',
@ -288,6 +288,11 @@ module.exports = OutputCacheManager = {
          pdfSize,
          timings.compile,
          function (err, result) {
+            if (err && err instanceof QueueLimitReachedError) {
+              logger.warn({ err, outputDir }, 'pdf caching queue limit reached')
+              stats['pdf-caching-queue-limit-reached'] = 1
+              return callback(null, outputFiles)
+            }
            if (err && err instanceof TimedOutError) {
              logger.warn(
                { err, outputDir, stats, timings },
--- a/services/clsi/buildscript.txt
+++ b/services/clsi/buildscript.txt
@ -2,7 +2,7 @@ clsi
 --data-dirs=cache,compiles,db,output
 --dependencies=
 --docker-repos=gcr.io/overleaf-ops
--env-add=ENABLE_PDF_CACHING="true"
+--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true"
 --env-pass-through=TEXLIVE_IMAGE
 --has-custom-cloudbuild=True
 --node-version=12.22.3
--- a/services/clsi/config/settings.defaults.js
+++ b/services/clsi/config/settings.defaults.js
@ -76,6 +76,12 @@ module.exports = {
    parseInt(process.env.PDF_CACHING_MIN_CHUNK_SIZE, 10) || 1024,
  pdfCachingMaxProcessingTime:
    parseInt(process.env.PDF_CACHING_MAX_PROCESSING_TIME, 10) || 10 * 1000,
+  pdfCachingEnableWorkerPool:
+    process.env.PDF_CACHING_ENABLE_WORKER_POOL === 'true',
+  pdfCachingWorkerPoolSize:
+    parseInt(process.env.PDF_CACHING_WORKER_POOL_SIZE, 10) || 4,
+  pdfCachingWorkerPoolBackLogLimit:
+    parseInt(process.env.PDF_CACHING_WORKER_POOL_BACK_LOG_LIMIT, 10) || 40,
 }

 if (process.env.ALLOWED_COMPILE_GROUPS) {
--- a/services/clsi/docker-compose.ci.yml
+++ b/services/clsi/docker-compose.ci.yml
@ -30,6 +30,7 @@ services:
      NODE_OPTIONS: "--unhandled-rejections=strict"
      TEXLIVE_IMAGE:
      ENABLE_PDF_CACHING: "true"
+      PDF_CACHING_ENABLE_WORKER_POOL: "true"
    command: npm run test:acceptance:_run


--- a/services/clsi/docker-compose.yml
+++ b/services/clsi/docker-compose.yml
@ -39,5 +39,6 @@ services:
      NODE_ENV: test
      NODE_OPTIONS: "--unhandled-rejections=strict"
      ENABLE_PDF_CACHING: "true"
+      PDF_CACHING_ENABLE_WORKER_POOL: "true"
    command: npm run --silent test:acceptance

--- a/services/clsi/package-lock.json
+++ b/services/clsi/package-lock.json
@ -1703,7 +1703,52 @@
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
    },
    "charenc": {
      "version": "0.0.2",
@ -1755,7 +1800,43 @@
      "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
      "dev": true,
      "requires": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0",
        "wrap-ansi": "^7.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        }
      }
    },
    "clone-response": {
@ -2246,14 +2327,40 @@
        "function-bind": "^1.1.1",
        "get-intrinsic": "^1.1.1",
        "has": "^1.0.3",
+        "has-symbols": "^1.0.2",
+        "is-callable": "^1.2.3",
        "is-negative-zero": "^2.0.1",
        "is-regex": "^1.1.3",
+        "is-string": "^1.0.6",
        "object-inspect": "^1.10.3",
        "object-keys": "^1.1.1",
        "object.assign": "^4.1.2",
        "string.prototype.trimend": "^1.0.4",
        "string.prototype.trimstart": "^1.0.4",
        "unbox-primitive": "^1.0.1"
+      },
+      "dependencies": {
+        "has-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
+          "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+          "dev": true
+        },
+        "is-callable": {
+          "version": "1.2.4",
+          "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz",
+          "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==",
+          "dev": true
+        },
+        "is-string": {
+          "version": "1.0.7",
+          "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
+          "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
+          "dev": true,
+          "requires": {
+            "has-tostringtag": "^1.0.0"
+          }
+        }
      }
    },
    "es-to-primitive": {
@ -2760,7 +2867,18 @@
      "version": "1.4.0",
      "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz",
      "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "estraverse": "^5.1.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+          "dev": true
+        }
+      }
    },
    "esrecurse": {
      "version": "4.3.0",
@ -2954,7 +3072,19 @@
      "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
      "dev": true,
      "requires": {
-        "flatted": "^3.1.0"
+        "flatted": "^3.1.0",
+        "rimraf": "^3.0.2"
+      },
+      "dependencies": {
+        "rimraf": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+          "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+          "dev": true,
+          "requires": {
+            "glob": "^7.1.3"
+          }
+        }
      }
    },
    "flatted": {
@ -3154,7 +3284,18 @@
      "version": "13.10.0",
      "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz",
      "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "type-fest": "^0.20.2"
+      },
+      "dependencies": {
+        "type-fest": {
+          "version": "0.20.2",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+          "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+          "dev": true
+        }
+      }
    },
    "google-auth-library": {
      "version": "6.0.6",
@ -3419,6 +3560,23 @@
      "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
      "dev": true
    },
+    "has-tostringtag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
+      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
+      "dev": true,
+      "requires": {
+        "has-symbols": "^1.0.2"
+      },
+      "dependencies": {
+        "has-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
+          "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+          "dev": true
+        }
+      }
+    },
    "has-unicode": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
@ -3771,7 +3929,16 @@
      "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==",
      "dev": true,
      "requires": {
-        "call-bind": "^1.0.2"
+        "call-bind": "^1.0.2",
+        "has-symbols": "^1.0.2"
+      },
+      "dependencies": {
+        "has-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
+          "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+          "dev": true
+        }
      }
    },
    "is-stream": {
@ -4333,6 +4500,7 @@
          "requires": {
            "anymatch": "~3.1.1",
            "braces": "~3.0.2",
+            "fsevents": "~2.3.1",
            "glob-parent": "~5.1.0",
            "is-binary-path": "~2.1.0",
            "is-glob": "~4.0.1",
@ -4386,6 +4554,13 @@
            "path-exists": "^4.0.0"
          }
        },
+        "fsevents": {
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+          "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+          "dev": true,
+          "optional": true
+        },
        "glob": {
          "version": "7.1.6",
          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
@ -4491,6 +4666,12 @@
          "requires": {
            "has-flag": "^4.0.0"
          }
+        },
+        "workerpool": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz",
+          "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==",
+          "dev": true
        }
      }
    },
@ -5850,13 +6031,38 @@
      "dev": true,
      "requires": {
        "ansi-styles": "^4.0.0",
-        "astral-regex": "^2.0.0"
+        "astral-regex": "^2.0.0",
+        "is-fullwidth-code-point": "^3.0.0"
      },
      "dependencies": {
        "ansi-styles": {
          "version": "4.3.0",
          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
          "dev": true
        }
      }
@ -6093,7 +6299,9 @@
        "ajv": "^8.0.1",
        "lodash.clonedeep": "^4.5.0",
        "lodash.truncate": "^4.4.2",
-        "slice-ansi": "^4.0.0"
+        "slice-ansi": "^4.0.0",
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0"
      },
      "dependencies": {
        "ajv": {
@ -6103,9 +6311,48 @@
          "dev": true,
          "requires": {
            "fast-deep-equal": "^3.1.1",
+            "json-schema-traverse": "^1.0.0",
            "require-from-string": "^2.0.2",
            "uri-js": "^4.2.2"
          }
+        },
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "json-schema-traverse": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+          "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
        }
      }
    },
@ -6671,16 +6918,78 @@
      "dev": true
    },
    "workerpool": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz",
-      "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==",
-      "dev": true
+      "version": "6.1.5",
+      "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.5.tgz",
+      "integrity": "sha512-XdKkCK0Zqc6w3iTxLckiuJ81tiD/o5rBE/m+nXpRCB+/Sq4DqkfXZ/x0jW02DG1tGsfUGXbTJyZDP+eu67haSw=="
    },
    "wrap-ansi": {
      "version": "7.0.0",
      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        }
+      }
    },
    "wrappy": {
      "version": "1.0.2",
@ -6731,8 +7040,43 @@
        "escalade": "^3.1.1",
        "get-caller-file": "^2.0.5",
        "require-directory": "^2.1.1",
+        "string-width": "^4.2.0",
        "y18n": "^5.0.5",
        "yargs-parser": "^20.2.2"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        }
      }
    },
    "yargs-parser": {
--- a/services/clsi/package.json
+++ b/services/clsi/package.json
@ -35,6 +35,7 @@
    "lodash": "^4.17.21",
    "logger-sharelatex": "^2.2.0",
    "mysql": "^2.18.1",
+    "workerpool": "^6.1.5",
    "p-limit": "^3.1.0",
    "pdfjs-dist": "^2.7.570",
    "request": "^2.88.2",