diff --git a/services/clsi/.dockerignore b/services/clsi/.dockerignore index bcbd758418..3202dcb3b0 100644 --- a/services/clsi/.dockerignore +++ b/services/clsi/.dockerignore @@ -7,5 +7,4 @@ gitrev nodemon.json cache/ compiles/ -db/ output/ diff --git a/services/clsi/.gitignore b/services/clsi/.gitignore index dee3eca1f5..360466227e 100644 --- a/services/clsi/.gitignore +++ b/services/clsi/.gitignore @@ -7,9 +7,6 @@ output *~ cache .vagrant -db.sqlite -db.sqlite-wal -db.sqlite-shm config/* npm-debug.log diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile index 8bd4b9c62f..28e81ec4b8 100644 --- a/services/clsi/Dockerfile +++ b/services/clsi/Dockerfile @@ -22,7 +22,7 @@ COPY . /app FROM base COPY --from=app /app /app -RUN mkdir -p cache compiles db output \ -&& chown node:node cache compiles db output +RUN mkdir -p cache compiles output \ +&& chown node:node cache compiles output CMD ["node", "--expose-gc", "app.js"] diff --git a/services/clsi/README.md b/services/clsi/README.md index c5cf1d17f3..1b0b490b74 100644 --- a/services/clsi/README.md +++ b/services/clsi/README.md @@ -30,7 +30,6 @@ The CLSI can be configured through the following environment variables: * `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds * `SENTRY_DSN` - Sentry [Data Source Name](https://docs.sentry.io/product/sentry-basics/dsn-explainer/) * `SMOKE_TEST` - Whether to run smoke tests -* `SQLITE_PATH` - Path to SQLite database * `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1` * `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `gcr.io/overleaf-ops` * `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex` diff --git a/services/clsi/app.js b/services/clsi/app.js index fbb2d4eba6..56b9e0838a 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -30,7 +30,6 @@ const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager') const OutputCacheManager = require('./app/js/OutputCacheManager') const ContentCacheManager = require('./app/js/ContentCacheManager') -require('./app/js/db').sync() ProjectPersistenceManager.init() const express = require('express') diff --git a/services/clsi/app/js/db.js b/services/clsi/app/js/db.js deleted file mode 100644 index 856f1da08b..0000000000 --- a/services/clsi/app/js/db.js +++ /dev/null @@ -1,67 +0,0 @@ -/* eslint-disable - no-console, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const Sequelize = require('sequelize') -const Settings = require('@overleaf/settings') -const _ = require('lodash') -const logger = require('logger-sharelatex') - -const options = _.extend({ logging: false }, Settings.mysql.clsi) - -logger.log({ dbPath: Settings.mysql.clsi.storage }, 'connecting to db') - -const sequelize = new Sequelize( - Settings.mysql.clsi.database, - Settings.mysql.clsi.username, - Settings.mysql.clsi.password, - options -) - -if (Settings.mysql.clsi.dialect === 'sqlite') { - logger.log('running PRAGMA journal_mode=WAL;') - sequelize.query('PRAGMA journal_mode=WAL;') - sequelize.query('PRAGMA synchronous=OFF;') - sequelize.query('PRAGMA read_uncommitted = true;') -} - -module.exports = { - UrlCache: sequelize.define( - 'UrlCache', - { - url: Sequelize.STRING, - project_id: Sequelize.STRING, - lastModified: Sequelize.DATE, - }, - { - indexes: [{ fields: ['url', 'project_id'] }, { fields: ['project_id'] }], - } - ), - - Project: sequelize.define( - 'Project', - { - project_id: { type: Sequelize.STRING, primaryKey: true }, - lastAccessed: Sequelize.DATE, - }, - { - indexes: [{ fields: ['lastAccessed'] }], - } - ), - - op: Sequelize.Op, - - sync() { - logger.log({ dbPath: Settings.mysql.clsi.storage }, 'syncing db schema') - return sequelize - .sync() - .then(() => logger.log('db sync complete')) - .catch(err => console.log(err, 'error syncing')) - }, -} diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index 8e5ca6e94e..ebadef0e4a 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -1,5 +1,5 @@ clsi ---data-dirs=cache,compiles,db,output +--data-dirs=cache,compiles,output --dependencies= --docker-repos=gcr.io/overleaf-ops --env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true" diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index a31e37dc3a..aa303eb6f5 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -1,25 +1,6 @@ const Path = require('path') module.exports = { - // Options are passed to Sequelize. - // See http://sequelizejs.com/documentation#usage-options for details - mysql: { - clsi: { - database: 'clsi', - username: 'clsi', - dialect: 'sqlite', - storage: - process.env.SQLITE_PATH || Path.resolve(__dirname, '../db/db.sqlite'), - pool: { - max: 1, - min: 1, - }, - retry: { - max: 10, - }, - }, - }, - compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb', processLifespanLimitMs: diff --git a/services/clsi/db/.gitignore b/services/clsi/db/.gitignore deleted file mode 100644 index d6b7ef32c8..0000000000 --- a/services/clsi/db/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/services/clsi/docker-compose-config.yml b/services/clsi/docker-compose-config.yml index 116c742bc3..9cc9cf4d98 100644 --- a/services/clsi/docker-compose-config.yml +++ b/services/clsi/docker-compose-config.yml @@ -23,7 +23,6 @@ services: SHARELATEX_CONFIG: /app/config/settings.defaults.js DOCKER_RUNNER: "true" COMPILES_HOST_DIR: $PWD/compiles - SQLITE_PATH: /app/compiles/db.sqlite volumes: - /var/run/docker.sock:/var/run/docker.sock:rw - ./compiles:/app/compiles diff --git a/services/clsi/package-lock.json b/services/clsi/package-lock.json index 80b2b250df..a12852b13f 100644 --- a/services/clsi/package-lock.json +++ b/services/clsi/package-lock.json @@ -1225,11 +1225,6 @@ "color-convert": "^1.9.0" } }, - "any-promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" - }, "anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -1437,11 +1432,6 @@ } } }, - "bluebird": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" - }, "body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", @@ -1848,15 +1838,6 @@ "mimic-response": "^1.0.0" } }, - "cls-bluebird": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.1.0.tgz", - "integrity": "sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4=", - "requires": { - "is-bluebird": "^1.0.2", - "shimmer": "^1.1.0" - } - }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -2214,11 +2195,6 @@ "is-obj": "^2.0.0" } }, - "dottie": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz", - "integrity": "sha512-fmrwR04lsniq/uSr8yikThDTrM7epXHBAAjH9TbeH3rEA8tdCO7mRzB9hdmdGyJCxF8KERo9CITcm3kGuoyMhg==" - }, "dtrace-provider": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", @@ -3737,11 +3713,6 @@ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, - "inflection": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz", - "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY=" - }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -3792,11 +3763,6 @@ "binary-extensions": "^2.0.0" } }, - "is-bluebird": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz", - "integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI=" - }, "is-boolean-object": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", @@ -4675,15 +4641,8 @@ "moment": { "version": "2.23.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.23.0.tgz", - "integrity": "sha1-dZ6kkayX1UusWtd2mW4qWMwbwiU=" - }, - "moment-timezone": { - "version": "0.5.28", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz", - "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==", - "requires": { - "moment": ">= 2.9.0" - } + "integrity": "sha1-dZ6kkayX1UusWtd2mW4qWMwbwiU=", + "optional": true }, "ms": { "version": "2.0.0", @@ -4725,38 +4684,6 @@ } } }, - "mysql": { - "version": "2.18.1", - "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", - "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", - "requires": { - "bignumber.js": "9.0.0", - "readable-stream": "2.3.7", - "safe-buffer": "5.1.2", - "sqlstring": "2.3.1" - }, - "dependencies": { - "bignumber.js": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", - "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - } - } - }, "nan": { "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", @@ -4780,16 +4707,6 @@ "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", "optional": true }, - "needle": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz", - "integrity": "sha1-UZMb/4JTOxkot9HWngHxsA/9Kk4=", - "requires": { - "debug": "^2.1.2", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - } - }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -4835,23 +4752,6 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" }, - "node-pre-gyp": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", - "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.1", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.2.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - } - }, "nodemon": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz", @@ -5751,14 +5651,6 @@ "lowercase-keys": "^1.0.0" } }, - "retry-as-promised": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz", - "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==", - "requires": { - "any-promise": "^1.3.0" - } - }, "retry-request": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", @@ -5871,58 +5763,6 @@ } } }, - "sequelize": { - "version": "5.21.5", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.5.tgz", - "integrity": "sha512-n9hR5K4uQGmBGK/Y/iqewCeSFmKVsd0TRnh0tfoLoAkmXbKC4tpeK96RhKs7d+TTMtrJlgt2TNLVBaAxEwC4iw==", - "requires": { - "bluebird": "^3.5.0", - "cls-bluebird": "^2.1.0", - "debug": "^4.1.1", - "dottie": "^2.0.0", - "inflection": "1.12.0", - "lodash": "^4.17.15", - "moment": "^2.24.0", - "moment-timezone": "^0.5.21", - "retry-as-promised": "^3.2.0", - "semver": "^6.3.0", - "sequelize-pool": "^2.3.0", - "toposort-class": "^1.0.1", - "uuid": "^3.3.3", - "validator": "^10.11.0", - "wkx": "^0.4.8" - }, - "dependencies": { - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "moment": { - "version": "2.24.0", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" - } - } - }, - "sequelize-pool": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz", - "integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA==" - }, "serialize-javascript": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", @@ -6139,21 +5979,6 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, - "sqlite3": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.1.1.tgz", - "integrity": "sha512-CvT5XY+MWnn0HkbwVKJAyWEMfzpAPwnTiB3TobA5Mri44SrTovmmh499NPQP+gatkeOipqPlBLel7rn4E/PCQg==", - "requires": { - "nan": "^2.12.1", - "node-pre-gyp": "^0.11.0", - "request": "^2.87.0" - } - }, - "sqlstring": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", - "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" - }, "ssh2": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.8.tgz", @@ -6507,11 +6332,6 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, - "toposort-class": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", - "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg=" - }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -6775,11 +6595,6 @@ "spdx-expression-parse": "^3.0.0" } }, - "validator": { - "version": "10.11.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz", - "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw==" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -6873,14 +6688,6 @@ } } }, - "wkx": { - "version": "0.4.8", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz", - "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==", - "requires": { - "@types/node": "*" - } - }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", diff --git a/services/clsi/package.json b/services/clsi/package.json index 33c22d871b..9a62770619 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -33,14 +33,11 @@ "lockfile": "^1.0.4", "lodash": "^4.17.21", "logger-sharelatex": "^2.2.0", - "mysql": "^2.18.1", - "workerpool": "^6.1.5", "p-limit": "^3.1.0", "pdfjs-dist": "^2.7.570", "request": "^2.88.2", "send": "^0.17.1", - "sequelize": "^5.21.5", - "sqlite3": "^4.1.1" + "workerpool": "^6.1.5" }, "devDependencies": { "chai": "^4.2.0", diff --git a/services/clsi/test/acceptance/scripts/settings.test.js b/services/clsi/test/acceptance/scripts/settings.test.js index 877ad34641..d6dda48107 100644 --- a/services/clsi/test/acceptance/scripts/settings.test.js +++ b/services/clsi/test/acceptance/scripts/settings.test.js @@ -1,18 +1,6 @@ const Path = require('path') module.exports = { - // Options are passed to Sequelize. - // See http://sequelizejs.com/documentation#usage-options for details - mysql: { - clsi: { - database: 'clsi', - username: 'clsi', - password: null, - dialect: 'sqlite', - storage: Path.resolve('db.sqlite'), - }, - }, - path: { // eslint-disable-next-line no-path-concat compilesDir: Path.resolve(__dirname + '/../../../compiles'), diff --git a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js index eceaf89821..a7ca9e48ba 100644 --- a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js +++ b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js @@ -35,7 +35,6 @@ describe('ProjectPersistenceManager', function () { clsiCacheDir: '/cache', }, }), - './db': (this.db = {}), }, }) this.callback = sinon.stub()