github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #5351 from overleaf/jpa-clsi-drop-sqlite

Browse source 
[clsi] goodbye sqlite db

GitOrigin-RevId: e7f16920c44e74a425b92884b48a60272dfb015b
This commit is contained in:
Jakob Ackermann 2021-10-06 10:44:05 +02:00 committed by Copybot
parent 03e1daa038
commit 94c208311c
14 changed files with 6 additions and 310 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
services/clsi/.dockerignore
View file

@ -7,5 +7,4 @@ gitrev
nodemon.json
cache/
compiles/
db/
output/

3
services/clsi/.gitignore vendored
View file

@ -7,9 +7,6 @@ output
*~
cache
.vagrant
db.sqlite
db.sqlite-wal
db.sqlite-shm
config/*
npm-debug.log

4
services/clsi/Dockerfile
View file

@ -22,7 +22,7 @@ COPY . /app
FROM base
COPY --from=app /app /app
RUN mkdir -p cache compiles db output \
&& chown node:node cache compiles db output
RUN mkdir -p cache compiles output \
&& chown node:node cache compiles output
CMD ["node", "--expose-gc", "app.js"]

1
services/clsi/README.md
View file

@ -30,7 +30,6 @@ The CLSI can be configured through the following environment variables:
* `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds
* `SENTRY_DSN` - Sentry [Data Source Name](https://docs.sentry.io/product/sentry-basics/dsn-explainer/)
* `SMOKE_TEST` - Whether to run smoke tests
* `SQLITE_PATH` - Path to SQLite database
* `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1`
* `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `gcr.io/overleaf-ops`
* `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex`

1
services/clsi/app.js
View file

@ -30,7 +30,6 @@ const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager')
const OutputCacheManager = require('./app/js/OutputCacheManager')
const ContentCacheManager = require('./app/js/ContentCacheManager')
require('./app/js/db').sync()
ProjectPersistenceManager.init()
const express = require('express')

67
services/clsi/app/js/db.js
View file

@ -1,67 +0,0 @@
/* eslint-disable
no-console,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const Sequelize = require('sequelize')
const Settings = require('@overleaf/settings')
const _ = require('lodash')
const logger = require('logger-sharelatex')
const options = _.extend({ logging: false }, Settings.mysql.clsi)
logger.log({ dbPath: Settings.mysql.clsi.storage }, 'connecting to db')
const sequelize = new Sequelize(
Settings.mysql.clsi.database,
Settings.mysql.clsi.username,
Settings.mysql.clsi.password,
options
)
if (Settings.mysql.clsi.dialect === 'sqlite') {
logger.log('running PRAGMA journal_mode=WAL;')
sequelize.query('PRAGMA journal_mode=WAL;')
sequelize.query('PRAGMA synchronous=OFF;')
sequelize.query('PRAGMA read_uncommitted = true;')
}
module.exports = {
UrlCache: sequelize.define(
'UrlCache',
{
url: Sequelize.STRING,
project_id: Sequelize.STRING,
lastModified: Sequelize.DATE,
},
{
indexes: [{ fields: ['url', 'project_id'] }, { fields: ['project_id'] }],
}
),
Project: sequelize.define(
'Project',
{
project_id: { type: Sequelize.STRING, primaryKey: true },
lastAccessed: Sequelize.DATE,
},
{
indexes: [{ fields: ['lastAccessed'] }],
}
),
op: Sequelize.Op,
sync() {
logger.log({ dbPath: Settings.mysql.clsi.storage }, 'syncing db schema')
return sequelize
.sync()
.then(() => logger.log('db sync complete'))
.catch(err => console.log(err, 'error syncing'))
},
}

2
services/clsi/buildscript.txt
View file

@ -1,5 +1,5 @@
clsi
--data-dirs=cache,compiles,db,output
--data-dirs=cache,compiles,output
--dependencies=
--docker-repos=gcr.io/overleaf-ops
--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true"

19
services/clsi/config/settings.defaults.js
View file

@ -1,25 +1,6 @@
const Path = require('path')
module.exports = {
// Options are passed to Sequelize.
// See http://sequelizejs.com/documentation#usage-options for details
mysql: {
clsi: {
database: 'clsi',
username: 'clsi',
dialect: 'sqlite',
storage:
process.env.SQLITE_PATH || Path.resolve(__dirname, '../db/db.sqlite'),
pool: {
max: 1,
min: 1,
},
retry: {
max: 10,
},
},
},
compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
processLifespanLimitMs:

2
services/clsi/db/.gitignore vendored
View file

@ -1,2 +0,0 @@
*
!.gitignore

1
services/clsi/docker-compose-config.yml
View file

@ -23,7 +23,6 @@ services:
SHARELATEX_CONFIG: /app/config/settings.defaults.js
DOCKER_RUNNER: "true"
COMPILES_HOST_DIR: $PWD/compiles
SQLITE_PATH: /app/compiles/db.sqlite
volumes:
- /var/run/docker.sock:/var/run/docker.sock:rw
- ./compiles:/app/compiles

197
services/clsi/package-lock.json generated
View file

@ -1225,11 +1225,6 @@
"color-convert": "^1.9.0"
}
},
"any-promise": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
"integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
},
"anymatch": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
@ -1437,11 +1432,6 @@
}
}
},
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"body-parser": {
"version": "1.19.0",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
@ -1848,15 +1838,6 @@
"mimic-response": "^1.0.0"
}
},
"cls-bluebird": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.1.0.tgz",
"integrity": "sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4=",
"requires": {
"is-bluebird": "^1.0.2",
"shimmer": "^1.1.0"
}
},
"code-point-at": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
@ -2214,11 +2195,6 @@
"is-obj": "^2.0.0"
}
},
"dottie": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",
"integrity": "sha512-fmrwR04lsniq/uSr8yikThDTrM7epXHBAAjH9TbeH3rEA8tdCO7mRzB9hdmdGyJCxF8KERo9CITcm3kGuoyMhg=="
},
"dtrace-provider": {
"version": "0.8.8",
"resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz",
@ -3737,11 +3713,6 @@
"integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
"dev": true
},
"inflection": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz",
"integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY="
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@ -3792,11 +3763,6 @@
"binary-extensions": "^2.0.0"
}
},
"is-bluebird": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz",
"integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI="
},
"is-boolean-object": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz",
@ -4675,15 +4641,8 @@
"moment": {
"version": "2.23.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.23.0.tgz",
"integrity": "sha1-dZ6kkayX1UusWtd2mW4qWMwbwiU="
},
"moment-timezone": {
"version": "0.5.28",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz",
"integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==",
"requires": {
"moment": ">= 2.9.0"
}
"integrity": "sha1-dZ6kkayX1UusWtd2mW4qWMwbwiU=",
"optional": true
},
"ms": {
"version": "2.0.0",
@ -4725,38 +4684,6 @@
}
}
},
"mysql": {
"version": "2.18.1",
"resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
"integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
"requires": {
"bignumber.js": "9.0.0",
"readable-stream": "2.3.7",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
},
"dependencies": {
"bignumber.js": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
"integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="
},
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
}
}
},
"nan": {
"version": "2.14.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
@ -4780,16 +4707,6 @@
"integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
"optional": true
},
"needle": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz",
"integrity": "sha1-UZMb/4JTOxkot9HWngHxsA/9Kk4=",
"requires": {
"debug": "^2.1.2",
"iconv-lite": "^0.4.4",
"sax": "^1.2.4"
}
},
"negotiator": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@ -4835,23 +4752,6 @@
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz",
"integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ=="
},
"node-pre-gyp": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
"integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
"requires": {
"detect-libc": "^1.0.2",
"mkdirp": "^0.5.1",
"needle": "^2.2.1",
"nopt": "^4.0.1",
"npm-packlist": "^1.1.6",
"npmlog": "^4.0.2",
"rc": "^1.2.7",
"rimraf": "^2.6.1",
"semver": "^5.3.0",
"tar": "^4"
}
},
"nodemon": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz",
@ -5751,14 +5651,6 @@
"lowercase-keys": "^1.0.0"
}
},
"retry-as-promised": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz",
"integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==",
"requires": {
"any-promise": "^1.3.0"
}
},
"retry-request": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz",
@ -5871,58 +5763,6 @@
}
}
},
"sequelize": {
"version": "5.21.5",
"resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.5.tgz",
"integrity": "sha512-n9hR5K4uQGmBGK/Y/iqewCeSFmKVsd0TRnh0tfoLoAkmXbKC4tpeK96RhKs7d+TTMtrJlgt2TNLVBaAxEwC4iw==",
"requires": {
"bluebird": "^3.5.0",
"cls-bluebird": "^2.1.0",
"debug": "^4.1.1",
"dottie": "^2.0.0",
"inflection": "1.12.0",
"lodash": "^4.17.15",
"moment": "^2.24.0",
"moment-timezone": "^0.5.21",
"retry-as-promised": "^3.2.0",
"semver": "^6.3.0",
"sequelize-pool": "^2.3.0",
"toposort-class": "^1.0.1",
"uuid": "^3.3.3",
"validator": "^10.11.0",
"wkx": "^0.4.8"
},
"dependencies": {
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"moment": {
"version": "2.24.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
}
}
},
"sequelize-pool": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz",
"integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA=="
},
"serialize-javascript": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz",
@ -6139,21 +5979,6 @@
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true
},
"sqlite3": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.1.1.tgz",
"integrity": "sha512-CvT5XY+MWnn0HkbwVKJAyWEMfzpAPwnTiB3TobA5Mri44SrTovmmh499NPQP+gatkeOipqPlBLel7rn4E/PCQg==",
"requires": {
"nan": "^2.12.1",
"node-pre-gyp": "^0.11.0",
"request": "^2.87.0"
}
},
"sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
},
"ssh2": {
"version": "0.8.8",
"resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.8.tgz",
@ -6507,11 +6332,6 @@
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
},
"toposort-class": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
"integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
},
"touch": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz",
@ -6775,11 +6595,6 @@
"spdx-expression-parse": "^3.0.0"
}
},
"validator": {
"version": "10.11.0",
"resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",
"integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw=="
},
"vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
@ -6873,14 +6688,6 @@
}
}
},
"wkx": {
"version": "0.4.8",
"resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz",
"integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==",
"requires": {
"@types/node": "*"
}
},
"word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",

5
services/clsi/package.json
View file

@ -33,14 +33,11 @@
"lockfile": "^1.0.4",
"lodash": "^4.17.21",
"logger-sharelatex": "^2.2.0",
"mysql": "^2.18.1",
"workerpool": "^6.1.5",
"p-limit": "^3.1.0",
"pdfjs-dist": "^2.7.570",
"request": "^2.88.2",
"send": "^0.17.1",
"sequelize": "^5.21.5",
"sqlite3": "^4.1.1"
"workerpool": "^6.1.5"
},
"devDependencies": {
"chai": "^4.2.0",

12
services/clsi/test/acceptance/scripts/settings.test.js
View file

@ -1,18 +1,6 @@
const Path = require('path')
module.exports = {
// Options are passed to Sequelize.
// See http://sequelizejs.com/documentation#usage-options for details
mysql: {
clsi: {
database: 'clsi',
username: 'clsi',
password: null,
dialect: 'sqlite',
storage: Path.resolve('db.sqlite'),
},
},
path: {
// eslint-disable-next-line no-path-concat
compilesDir: Path.resolve(__dirname + '/../../../compiles'),

1
services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
View file

@ -35,7 +35,6 @@ describe('ProjectPersistenceManager', function () {
clsiCacheDir: '/cache',
},
}),
'./db': (this.db = {}),
},
})
this.callback = sinon.stub()