Merge pull request #4819 from overleaf/jpa-pdf-caching-new-event-loop

[clsi] pdf-caching: move cpu intensive work onto a new event loop

GitOrigin-RevId: 4cb5cd4528fa1c5df6a8e91f9caa38cb64d94463

services/clsi/app/js/ContentCacheManager.js

@@ -10,7 +10,36 @@ const Settings = require('@overleaf/settings')
 const OError = require('@overleaf/o-error')
 const pLimit = require('p-limit')
 const { parseXrefTable } = require('../lib/pdfjs/parseXrefTable')
-const { TimedOutError } = require('./Errors')
+const { QueueLimitReachedError, TimedOutError } = require('./Errors')
+const workerpool = require('workerpool')
+const Metrics = require('@overleaf/metrics')
+
+let WORKER_POOL
+// NOTE: Check for main thread to avoid recursive start of pool.
+if (Settings.pdfCachingEnableWorkerPool && workerpool.isMainThread) {
+  WORKER_POOL = workerpool.pool(Path.join(__dirname, 'ContentCacheWorker.js'), {
+    // Cap number of worker threads.
+    maxWorkers: Settings.pdfCachingWorkerPoolSize,
+    // Warmup workers.
+    minWorkers: Settings.pdfCachingWorkerPoolSize,
+    // Limit queue back-log
+    maxQueueSize: Settings.pdfCachingWorkerPoolBackLogLimit,
+  })
+  setInterval(() => {
+    const {
+      totalWorkers,
+      busyWorkers,
+      idleWorkers,
+      pendingTasks,
+      activeTasks,
+    } = WORKER_POOL.stats()
+    Metrics.gauge('pdf_caching_total_workers', totalWorkers)
+    Metrics.gauge('pdf_caching_busy_workers', busyWorkers)
+    Metrics.gauge('pdf_caching_idle_workers', idleWorkers)
+    Metrics.gauge('pdf_caching_pending_tasks', pendingTasks)
+    Metrics.gauge('pdf_caching_active_tasks', activeTasks)
+  }, 15 * 1000)
+}
 
 /**
  *
@@ -20,7 +49,68 @@ const { TimedOutError } = require('./Errors')
  * @param {number} compileTime
  */
 async function update(contentDir, filePath, size, compileTime) {
+  if (Settings.pdfCachingEnableWorkerPool) {
+    return await updateOtherEventLoop(contentDir, filePath, size, compileTime)
+  } else {
+    return await updateSameEventLoop(contentDir, filePath, size, compileTime)
+  }
+}
+
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ * @param {number} compileTime
+ */
+async function updateOtherEventLoop(contentDir, filePath, size, compileTime) {
+  const timeout = getMaxOverhead(compileTime)
+  try {
+    return await WORKER_POOL.exec('doUpdateInternalNoDeadline', [
+      contentDir,
+      filePath,
+      size,
+    ]).timeout(timeout)
+  } catch (e) {
+    if (e instanceof workerpool.Promise.TimeoutError) {
+      throw new TimedOutError('context-lost-in-worker')
+    }
+    if (e.message.includes('Max queue size of ')) {
+      throw new QueueLimitReachedError()
+    }
+    throw e
+  }
+}
+
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ * @param {number} compileTime
+ */
+async function updateSameEventLoop(contentDir, filePath, size, compileTime) {
   const checkDeadline = getDeadlineChecker(compileTime)
+  return doUpdateInternal(contentDir, filePath, size, checkDeadline)
+}
+
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ */
+async function doUpdateInternalNoDeadline(contentDir, filePath, size) {
+  return doUpdateInternal(contentDir, filePath, size, () => {})
+}
+/**
+ *
+ * @param {String} contentDir path to directory where content hash files are cached
+ * @param {String} filePath the pdf file to scan for streams
+ * @param {number} size the pdf size
+ * @param {function} checkDeadline
+ */
+async function doUpdateInternal(contentDir, filePath, size, checkDeadline) {
   const ranges = []
   const newRanges = []
   // keep track of hashes expire old ones when they reach a generation > N.
@@ -233,14 +323,18 @@ async function writePdfStream(dir, hash, buffer) {
   }
 }
 
-function getDeadlineChecker(compileTime) {
+function getMaxOverhead(compileTime) {
-  const maxOverhead = Math.min(
+  return Math.min(
     // Adding 10s to a  40s compile time is OK.
     // Adding  1s to a   3s compile time is OK.
     Math.max(compileTime / 4, 1000),
     // Adding 30s to a 120s compile time is not OK, limit to 10s.
     Settings.pdfCachingMaxProcessingTime
   )
+}
+
+function getDeadlineChecker(compileTime) {
+  const maxOverhead = getMaxOverhead(compileTime)
 
   const deadline = Date.now() + maxOverhead
   let lastStage = { stage: 'start', now: Date.now() }
@@ -269,5 +363,6 @@ module.exports = {
   update: callbackify(update),
   promises: {
     update,
+    doUpdateInternalNoDeadline,
   },
 }

services/clsi/app/js/ContentCacheMetrics.js

@@ -22,6 +22,9 @@ function emitPdfStats(stats, timings) {
   if (stats['pdf-caching-timed-out']) {
     Metrics.inc('pdf-caching-timed-out')
   }
+  if (stats['pdf-caching-queue-limit-reached']) {
+    Metrics.inc('pdf-caching-queue-limit-reached')
+  }
   if (timings['compute-pdf-caching']) {
     emitPdfCachingStats(stats, timings)
   } else {

services/clsi/app/js/ContentCacheWorker.js

@@ -0,0 +1,4 @@
+const workerpool = require('workerpool')
+const ContentCacheManager = require('./ContentCacheManager')
+
+workerpool.worker(ContentCacheManager.promises)

services/clsi/app/js/Errors.js

@@ -31,9 +31,11 @@ var AlreadyCompilingError = function (message) {
 }
 AlreadyCompilingError.prototype.__proto__ = Error.prototype
 
+class QueueLimitReachedError extends OError {}
 class TimedOutError extends OError {}
 
 module.exports = Errors = {
+  QueueLimitReachedError,
   TimedOutError,
   NotFoundError,
   FilesOutOfSyncError,

services/clsi/app/js/OutputCacheManager.js

@@ -26,7 +26,7 @@ const Metrics = require('./Metrics')
 
 const OutputFileOptimiser = require('./OutputFileOptimiser')
 const ContentCacheManager = require('./ContentCacheManager')
-const { TimedOutError } = require('./Errors')
+const { QueueLimitReachedError, TimedOutError } = require('./Errors')
 
 module.exports = OutputCacheManager = {
   CONTENT_SUBDIR: 'content',
@@ -288,6 +288,11 @@ module.exports = OutputCacheManager = {
           pdfSize,
           timings.compile,
           function (err, result) {
+            if (err && err instanceof QueueLimitReachedError) {
+              logger.warn({ err, outputDir }, 'pdf caching queue limit reached')
+              stats['pdf-caching-queue-limit-reached'] = 1
+              return callback(null, outputFiles)
+            }
             if (err && err instanceof TimedOutError) {
               logger.warn(
                 { err, outputDir, stats, timings },

services/clsi/buildscript.txt

@@ -2,7 +2,7 @@ clsi
 --data-dirs=cache,compiles,db,output
 --dependencies=
 --docker-repos=gcr.io/overleaf-ops
---env-add=ENABLE_PDF_CACHING="true"
+--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true"
 --env-pass-through=TEXLIVE_IMAGE
 --has-custom-cloudbuild=True
 --node-version=12.22.3

services/clsi/config/settings.defaults.js

@@ -76,6 +76,12 @@ module.exports = {
     parseInt(process.env.PDF_CACHING_MIN_CHUNK_SIZE, 10) || 1024,
   pdfCachingMaxProcessingTime:
     parseInt(process.env.PDF_CACHING_MAX_PROCESSING_TIME, 10) || 10 * 1000,
+  pdfCachingEnableWorkerPool:
+    process.env.PDF_CACHING_ENABLE_WORKER_POOL === 'true',
+  pdfCachingWorkerPoolSize:
+    parseInt(process.env.PDF_CACHING_WORKER_POOL_SIZE, 10) || 4,
+  pdfCachingWorkerPoolBackLogLimit:
+    parseInt(process.env.PDF_CACHING_WORKER_POOL_BACK_LOG_LIMIT, 10) || 40,
 }
 
 if (process.env.ALLOWED_COMPILE_GROUPS) {

services/clsi/docker-compose.ci.yml

@@ -30,6 +30,7 @@ services:
       NODE_OPTIONS: "--unhandled-rejections=strict"
       TEXLIVE_IMAGE:
       ENABLE_PDF_CACHING: "true"
+      PDF_CACHING_ENABLE_WORKER_POOL: "true"
     command: npm run test:acceptance:_run
 
 

services/clsi/docker-compose.yml

@@ -39,5 +39,6 @@ services:
       NODE_ENV: test
       NODE_OPTIONS: "--unhandled-rejections=strict"
       ENABLE_PDF_CACHING: "true"
+      PDF_CACHING_ENABLE_WORKER_POOL: "true"
     command: npm run --silent test:acceptance
 

services/clsi/package-lock.json

@@ -1703,7 +1703,52 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
       "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
     },
     "charenc": {
       "version": "0.0.2",
@@ -1755,7 +1800,43 @@
       "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
       "dev": true,
       "requires": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0",
         "wrap-ansi": "^7.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        }
       }
     },
     "clone-response": {
@@ -2246,14 +2327,40 @@
         "function-bind": "^1.1.1",
         "get-intrinsic": "^1.1.1",
         "has": "^1.0.3",
+        "has-symbols": "^1.0.2",
+        "is-callable": "^1.2.3",
         "is-negative-zero": "^2.0.1",
         "is-regex": "^1.1.3",
+        "is-string": "^1.0.6",
         "object-inspect": "^1.10.3",
         "object-keys": "^1.1.1",
         "object.assign": "^4.1.2",
         "string.prototype.trimend": "^1.0.4",
         "string.prototype.trimstart": "^1.0.4",
         "unbox-primitive": "^1.0.1"
+      },
+      "dependencies": {
+        "has-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
+          "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+          "dev": true
+        },
+        "is-callable": {
+          "version": "1.2.4",
+          "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz",
+          "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==",
+          "dev": true
+        },
+        "is-string": {
+          "version": "1.0.7",
+          "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
+          "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
+          "dev": true,
+          "requires": {
+            "has-tostringtag": "^1.0.0"
+          }
+        }
       }
     },
     "es-to-primitive": {
@@ -2760,7 +2867,18 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz",
       "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "estraverse": "^5.1.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+          "dev": true
+        }
+      }
     },
     "esrecurse": {
       "version": "4.3.0",
@@ -2954,7 +3072,19 @@
       "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
       "dev": true,
       "requires": {
-        "flatted": "^3.1.0"
+        "flatted": "^3.1.0",
+        "rimraf": "^3.0.2"
+      },
+      "dependencies": {
+        "rimraf": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+          "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+          "dev": true,
+          "requires": {
+            "glob": "^7.1.3"
+          }
+        }
       }
     },
     "flatted": {
@@ -3154,7 +3284,18 @@
       "version": "13.10.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz",
       "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "type-fest": "^0.20.2"
+      },
+      "dependencies": {
+        "type-fest": {
+          "version": "0.20.2",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+          "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+          "dev": true
+        }
+      }
     },
     "google-auth-library": {
       "version": "6.0.6",
@@ -3419,6 +3560,23 @@
       "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==",
       "dev": true
     },
+    "has-tostringtag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
+      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
+      "dev": true,
+      "requires": {
+        "has-symbols": "^1.0.2"
+      },
+      "dependencies": {
+        "has-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
+          "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+          "dev": true
+        }
+      }
+    },
     "has-unicode": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
@@ -3771,7 +3929,16 @@
       "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==",
       "dev": true,
       "requires": {
-        "call-bind": "^1.0.2"
+        "call-bind": "^1.0.2",
+        "has-symbols": "^1.0.2"
+      },
+      "dependencies": {
+        "has-symbols": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
+          "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+          "dev": true
+        }
       }
     },
     "is-stream": {
@@ -4333,6 +4500,7 @@
           "requires": {
             "anymatch": "~3.1.1",
             "braces": "~3.0.2",
+            "fsevents": "~2.3.1",
             "glob-parent": "~5.1.0",
             "is-binary-path": "~2.1.0",
             "is-glob": "~4.0.1",
@@ -4386,6 +4554,13 @@
             "path-exists": "^4.0.0"
           }
         },
+        "fsevents": {
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+          "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+          "dev": true,
+          "optional": true
+        },
         "glob": {
           "version": "7.1.6",
           "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
@@ -4491,6 +4666,12 @@
           "requires": {
             "has-flag": "^4.0.0"
           }
+        },
+        "workerpool": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz",
+          "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==",
+          "dev": true
         }
       }
     },
@@ -5850,13 +6031,38 @@
       "dev": true,
       "requires": {
         "ansi-styles": "^4.0.0",
-        "astral-regex": "^2.0.0"
+        "astral-regex": "^2.0.0",
+        "is-fullwidth-code-point": "^3.0.0"
       },
       "dependencies": {
         "ansi-styles": {
           "version": "4.3.0",
           "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
           "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
           "dev": true
         }
       }
@@ -6093,7 +6299,9 @@
         "ajv": "^8.0.1",
         "lodash.clonedeep": "^4.5.0",
         "lodash.truncate": "^4.4.2",
-        "slice-ansi": "^4.0.0"
+        "slice-ansi": "^4.0.0",
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0"
       },
       "dependencies": {
         "ajv": {
@@ -6103,9 +6311,48 @@
           "dev": true,
           "requires": {
             "fast-deep-equal": "^3.1.1",
+            "json-schema-traverse": "^1.0.0",
             "require-from-string": "^2.0.2",
             "uri-js": "^4.2.2"
           }
+        },
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "json-schema-traverse": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+          "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
         }
       }
     },
@@ -6671,16 +6918,78 @@
       "dev": true
     },
     "workerpool": {
-      "version": "6.1.0",
+      "version": "6.1.5",
-      "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.5.tgz",
-      "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==",
+      "integrity": "sha512-XdKkCK0Zqc6w3iTxLckiuJ81tiD/o5rBE/m+nXpRCB+/Sq4DqkfXZ/x0jW02DG1tGsfUGXbTJyZDP+eu67haSw=="
-      "dev": true
     },
     "wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
       "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        }
+      }
     },
     "wrappy": {
       "version": "1.0.2",
@@ -6731,8 +7040,43 @@
         "escalade": "^3.1.1",
         "get-caller-file": "^2.0.5",
         "require-directory": "^2.1.1",
+        "string-width": "^4.2.0",
         "y18n": "^5.0.5",
         "yargs-parser": "^20.2.2"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+          "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+          "dev": true
+        },
+        "string-width": {
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+          "dev": true,
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+          "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^5.0.0"
+          }
+        }
       }
     },
     "yargs-parser": {

services/clsi/package.json

@@ -35,6 +35,7 @@
     "lodash": "^4.17.21",
     "logger-sharelatex": "^2.2.0",
     "mysql": "^2.18.1",
+    "workerpool": "^6.1.5",
     "p-limit": "^3.1.0",
     "pdfjs-dist": "^2.7.570",
     "request": "^2.88.2",