github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-14 20:40:17 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #913 from overleaf/jpa-goodbye-grunt

Browse source 
[misc] goodbye grunt
This commit is contained in:
Jakob Ackermann 2021-07-14 17:23:28 +02:00 committed by GitHub
commit b1ad447c33
13 changed files with 113 additions and 3708 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
server-ce/Dockerfile
View file

@ -5,57 +5,32 @@
ARG SHARELATEX_BASE_TAG=sharelatex/sharelatex-base:latest
FROM $SHARELATEX_BASE_TAG
ENV SHARELATEX_CONFIG /etc/sharelatex/settings.js
WORKDIR /var/www/sharelatex
# Add required source files
# -------------------------
ADD ${baseDir}/bin /var/www/sharelatex/bin
ADD ${baseDir}/doc /var/www/sharelatex/doc
ADD ${baseDir}/tasks /var/www/sharelatex/tasks
ADD ${baseDir}/Gruntfile.js /var/www/sharelatex/Gruntfile.js
ADD ${baseDir}/package.json /var/www/sharelatex/package.json
ADD ${baseDir}/package-lock.json /var/www/sharelatex/package-lock.json
ADD ${baseDir}/services.js /var/www/sharelatex/config/services.js
# Copy build dependencies
# -----------------------
ADD ${baseDir}/git-revision.sh /var/www/git-revision.sh
ADD ${baseDir}/services.js /var/www/sharelatex/config/services.js
ADD ${baseDir}/genScript.js /var/www/sharelatex/genScript.js
ADD ${baseDir}/services.js /var/www/sharelatex/services.js
# Checkout services
# -----------------
RUN cd /var/www/sharelatex \
&& npm ci \
&& grunt install \
RUN node genScript checkout | bash \
\
# Cleanup not needed artifacts
# ----------------------------
&& rm -rf /root/.cache /root/.npm $(find /tmp/ -mindepth 1 -maxdepth 1) \
# Stores the version installed for each service
# Store the revision for each service
# ---------------------------------------------
&& cd /var/www \
&& ./git-revision.sh > revisions.txt \
&& node genScript revisions | bash > revisions.txt \
\
# Cleanup the git history
# -------------------
&& rm -rf $(find /var/www/sharelatex -name .git)
&& node genScript cleanup-git | bash
# Install npm dependencies
# ------------------------
RUN cd /var/www/sharelatex \
&& bash ./bin/install-services \
\
# Cleanup not needed artifacts
# ----------------------------
&& rm -rf /root/.cache /root/.npm $(find /tmp/ -mindepth 1 -maxdepth 1)
RUN node genScript install | bash
# Compile CoffeeScript
# Compile
# --------------------
RUN cd /var/www/sharelatex \
&& bash ./bin/compile-services
RUN node genScript compile | bash
# Links CLSI synctex to its default location
# ------------------------------------------
@ -87,8 +62,15 @@ COPY ${baseDir}/init_scripts/ /etc/my_init.d/
# -----------------------
COPY ${baseDir}/settings.js /etc/sharelatex/settings.js
# Copy grunt thin wrapper
# -----------------------
ADD ${baseDir}/bin/grunt /usr/local/bin/grunt
RUN chmod +x /usr/local/bin/grunt
# Set Environment Variables
# --------------------------------
ENV SHARELATEX_CONFIG /etc/sharelatex/settings.js
ENV WEB_API_USER "sharelatex"
ENV SHARELATEX_APP_NAME "Overleaf Community Edition"

6
server-ce/Dockerfile-base
View file

@ -35,12 +35,6 @@ RUN apt-get update \
ADD ./vendor/envsubst /usr/bin/envsubst
RUN chmod +x /usr/bin/envsubst
# Install Grunt
# ------------
RUN npm install -g \
grunt-cli \
&& rm -rf /root/.npm
# Install TexLive
# ---------------
# CTAN mirrors occasionally fail, in that case install TexLive against an

355
server-ce/Gruntfile.js
View file

@ -1,355 +0,0 @@
/* eslint-disable
camelcase,
no-return-assign,
no-unreachable,
no-unused-vars,
node/handle-callback-err,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__, or convert again using --optional-chaining
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const fs = require('fs')
const { spawn } = require('child_process')
const { exec } = require('child_process')
const rimraf = require('rimraf')
const Path = require('path')
const semver = require('semver')
const knox = require('knox')
const crypto = require('crypto')
const async = require('async')
const settings = require('@overleaf/settings')
const _ = require('underscore')
const SERVICES = require('./config/services')
module.exports = function (grunt) {
let Helpers
let service
grunt.loadNpmTasks('grunt-bunyan')
grunt.loadNpmTasks('grunt-execute')
grunt.loadNpmTasks('grunt-available-tasks')
grunt.loadNpmTasks('grunt-concurrent')
grunt.loadNpmTasks('grunt-shell')
grunt.task.loadTasks('./tasks')
const execute = {}
for (service of Array.from(SERVICES)) {
execute[service.name] = { src: `${service.name}/app.js` }
}
grunt.initConfig({
execute,
concurrent: {
all: {
tasks: (() => {
const result = []
for (service of Array.from(SERVICES)) {
result.push(`run:${service.name}`)
}
return result
})(),
options: {
limit: SERVICES.length,
logConcurrentOutput: true,
},
},
},
availabletasks: {
tasks: {
options: {
filter: 'exclude',
tasks: ['concurrent', 'execute', 'bunyan', 'availabletasks'],
groups: {
'Run tasks': ['run', 'run:all', 'default'].concat(
(() => {
const result1 = []
for (service of Array.from(SERVICES)) {
result1.push(`run:${service.name}`)
}
return result1
})()
),
Misc: ['help'],
'Install tasks': (() => {
const result2 = []
for (service of Array.from(SERVICES)) {
result2.push(`install:${service.name}`)
}
return result2
})().concat(['install:all', 'install']),
'Update tasks': (() => {
const result3 = []
for (service of Array.from(SERVICES)) {
result3.push(`update:${service.name}`)
}
return result3
})().concat(['update:all', 'update']),
Checks: [
'check',
'check:redis',
'check:latexmk',
'check:s3',
'check:make',
'check:mongo',
],
},
},
},
},
})
for (service of Array.from(SERVICES)) {
;(service =>
grunt.registerTask(
`install:${service.name}`,
`Download and set up the ${service.name} service`,
function () {
const done = this.async()
return Helpers.installService(service, done)
}
))(service)
}
grunt.registerTask(
'install:all',
'Download and set up all ShareLaTeX services',
[]
.concat(
(() => {
const result4 = []
for (service of Array.from(SERVICES)) {
result4.push(`install:${service.name}`)
}
return result4
})()
)
.concat(['postinstall'])
)
grunt.registerTask('install', 'install:all')
grunt.registerTask('postinstall', 'Explain postinstall steps', function () {
return Helpers.postinstallMessage(this.async())
})
grunt.registerTask(
'update:all',
'Checkout and update all ShareLaTeX services',
['check:make'].concat(
(() => {
const result5 = []
for (service of Array.from(SERVICES)) {
result5.push(`update:${service.name}`)
}
return result5
})()
)
)
grunt.registerTask('update', 'update:all')
grunt.registerTask('run', 'Run all of the sharelatex processes', [
'concurrent:all',
])
grunt.registerTask('run:all', 'run')
grunt.registerTask('help', 'Display this help list', 'availabletasks')
grunt.registerTask('default', 'run')
grunt.registerTask(
'check:redis',
'Check that redis is installed and running',
function () {
return Helpers.checkRedisConnect(this.async())
}
)
grunt.registerTask(
'check:mongo',
'Check that mongo is installed',
function () {
return Helpers.checkMongoConnect(this.async())
}
)
grunt.registerTask(
'check',
'Check that you have the required dependencies installed',
['check:redis', 'check:mongo', 'check:make']
)
grunt.registerTask('check:make', 'Check that make is installed', function () {
return Helpers.checkMake(this.async())
})
return (Helpers = {
installService(service, callback) {
if (callback == null) {
callback = function (error) {}
}
console.log(`Installing ${service.name}`)
return Helpers.cloneGitRepo(service, function (error) {
if (error != null) {
return callback(error)
} else {
return callback()
}
})
},
cloneGitRepo(service, callback) {
if (callback == null) {
callback = function (error) {}
}
const repo_src = service.repo
const dir = service.name
if (!fs.existsSync(dir)) {
const proc = spawn('git', ['clone', repo_src, dir], {
stdio: 'inherit',
})
return proc.on('close', () =>
Helpers.checkoutVersion(service, callback)
)
} else {
console.log(`${dir} already installed, skipping.`)
return callback()
}
},
checkoutVersion(service, callback) {
if (callback == null) {
callback = function (error) {}
}
const dir = service.name
grunt.log.write(`checking out ${service.name} ${service.version}`)
const proc = spawn('git', ['checkout', service.version], {
stdio: 'inherit',
cwd: dir,
})
return proc.on('close', () => callback())
},
postinstallMessage(callback) {
if (callback == null) {
callback = function (error) {}
}
grunt.log.write(`\
Services cloned:
${(() => {
const result6 = []
for (service of Array.from(SERVICES)) {
result6.push(service.name)
}
return result6
})()}
To install services run:
$ source bin/install-services
This will install the required node versions and run \`npm install\` for each service.
See https://github.com/sharelatex/sharelatex/pull/549 for more info.\
`)
return callback()
},
checkMake(callback) {
if (callback == null) {
callback = function (error) {}
}
grunt.log.write('Checking make is installed... ')
return exec('make --version', function (error, stdout, stderr) {
if (error != null && error.message.match('not found')) {
grunt.log.error('FAIL.')
grunt.log.errorlns(`\
Either make is not installed or is not in your path.
On Ubuntu you can install make with:
sudo apt-get install build-essential
\
`)
return callback(error)
} else if (error != null) {
return callback(error)
} else {
grunt.log.write('OK.')
return callback()
}
})
},
checkMongoConnect(callback) {
if (callback == null) {
callback = function (error) {}
}
grunt.log.write('Checking can connect to mongo')
const mongojs = require('mongojs')
const db = mongojs(settings.mongo.url, ['tags'])
db.runCommand({ ping: 1 }, function (err, res) {
if (!err && res.ok) {
grunt.log.write('OK.')
}
return callback()
})
return db.on('error', function (err) {
err = 'Can not connect to mongodb'
grunt.log.error('FAIL.')
grunt.log.errorlns(`\
!!!!!!!!!!!!!! MONGO ERROR !!!!!!!!!!!!!!
ShareLaTeX can not talk to the mongodb instance
Check the mongodb instance is running and accessible on env var SHARELATEX_MONGO_URL
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\
`)
throw new Error('Can not connect to Mongodb')
return callback(err)
})
},
checkRedisConnect(callback) {
if (callback == null) {
callback = function (error) {}
}
grunt.log.write('Checking can connect to redis\n')
const rclient = require('redis').createClient(settings.redis.web)
rclient.ping(function (err, res) {
if (err == null) {
grunt.log.write('OK.')
} else {
throw new Error('Can not connect to redis')
}
return callback()
})
const errorHandler = _.once(function (err) {
err = 'Can not connect to redis'
grunt.log.error('FAIL.')
grunt.log.errorlns(`\
!!!!!!!!!!!!!! REDIS ERROR !!!!!!!!!!!!!!
ShareLaTeX can not talk to the redis instance
Check the redis instance is running and accessible on env var SHARELATEX_REDIS_HOST
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\
`)
throw new Error('Can not connect to redis')
return callback(err)
})
return rclient.on('error', errorHandler)
},
})
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

2
server-ce/README.md
View file

@ -42,7 +42,7 @@ If you are upgrading from a previous version of Overleaf, please see the [Releas
## Other repositories
This repository does not contain any code. It acts a wrapper and toolkit for managing the many different Overleaf services. These each run as their own Node.js process and have their own GitHub repository. These are all downloaded and set up when you run `grunt install`
This repository does not contain any code. It acts a wrapper and toolkit for managing the many different Overleaf services. These each run as their own Node.js process and have their own GitHub repository.
| Service | Description |
| ------- | ----------- |

19
server-ce/bin/compile-services
View file

@ -1,19 +0,0 @@
#! env bash
set -e
node ./config/services.js | \
while read service
do
pushd $service
echo "Compiling Service $service"
case $service in
web)
npm run webpack:production
;;
*)
echo "$service doesn't require a compilation"
;;
esac
popd
done

31
server-ce/bin/grunt Executable file
View file

@ -0,0 +1,31 @@
#!/bin/bash
# Thin wrapper on old grunt tasks to ease migrating.
set -e
TASK="$1"
shift 1
cd /var/www/sharelatex/web/modules/server-ce-scripts/scripts
case "$TASK" in
user:create-admin)
node create-admin "$@"
;;
user:delete)
node delete-user "$@"
;;
check:mongo)
node check-mongodb
;;
check:redis)
node check-redis
;;
*)
echo "Unknown task $TASK"
exit 1
;;
esac

21
server-ce/bin/install-services
View file

@ -1,21 +0,0 @@
#! env bash
set -e
node ./config/services.js | \
while read service
do
pushd $service
echo "Installing service $service"
case $service in
web)
# install webpack and friends from dev-dependencies.
npm ci
;;
*)
npm ci --only=production
;;
esac
popd
done

61
server-ce/genScript.js Normal file
View file

@ -0,0 +1,61 @@
const services = require('./services')
console.log('#!/bin/bash')
console.log('set -ex')
switch (process.argv.pop()) {
case 'checkout':
for (const service of services) {
console.log(`git clone ${service.repo} ${service.name}`)
console.log(`git -C ${service.name} checkout ${service.version}`)
}
break
case 'revisions':
for (const service of services) {
console.log(`git -C ${service.name} rev-parse HEAD`)
}
break
case 'cleanup-git':
for (const service of services) {
console.log(`rm -rf ${service.name}/.git`)
}
break
case 'install':
for (const service of services) {
console.log('pushd', service.name)
switch (service.name) {
case 'web':
console.log('npm ci')
break
default:
// TODO(das7pad): revert back to npm ci --only=production (https://github.com/overleaf/issues/issues/4544)
console.log('npm ci')
}
console.log('popd')
}
break
case 'compile':
for (const service of services) {
console.log('pushd', service.name)
switch (service.name) {
case 'web':
console.log('npm run webpack:production')
// drop webpack/babel cache
console.log('rm -rf node_modules/.cache')
break
default:
console.log(`echo ${service.name} does not require a compilation`)
}
console.log('popd')
}
break
default:
console.error('unknown command')
console.log('exit 101')
process.exit(101)
}
console.log('set +x')
console.log(
'rm -rf /root/.cache /root/.npm $(find /tmp/ -mindepth 1 -maxdepth 1)'
)

6
server-ce/git-revision.sh
View file

@ -1,6 +0,0 @@
#!/bin/sh
for gitDir in $(find "$PWD" -name .git); do
echo -n "$(dirname ${gitDir}),"
git --git-dir="$gitDir" rev-parse HEAD
done

5
server-ce/init_scripts/98_check_db_access.sh
View file

@ -2,6 +2,7 @@
set -e
echo "Checking can connect to mongo and redis"
cd /var/www/sharelatex && grunt check:redis
cd /var/www/sharelatex && grunt check:mongo
cd /var/www/sharelatex/web/modules/server-ce-scripts/scripts
node check-mongodb
node check-redis
echo "All checks passed"

3100
server-ce/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

47
server-ce/package.json
View file

@ -1,47 +0,0 @@
{
"name": "sharelatex",
"version": "0.0.1",
"description": "An online collaborative LaTeX editor",
"scripts": {
"lint": "eslint --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix .",
"format": "prettier --list-different $PWD/'**/*.js'",
"format:fix": "prettier --write $PWD/'**/*.js'"
},
"dependencies": {
"@overleaf/settings": "^2.1.1",
"async": "^0.9.0",
"bson": "^1.0.4",
"east": "0.5.7",
"east-mongo": "0.3.3",
"grunt-shell": "^1.1.1",
"load-grunt-config": "^0.19.2",
"lodash": "^3.0.0",
"mongodb": "^2.2.34",
"mongojs": "2.4.0",
"redis": "^2.6.2",
"rimraf": "~2.2.6",
"underscore": "^1.7.0"
},
"devDependencies": {
"grunt": "~0.4.2",
"bunyan": "~0.22.1",
"eslint": "^7.21.0",
"eslint-config-prettier": "^8.1.0",
"eslint-config-standard": "^16.0.2",
"eslint-plugin-chai-expect": "^2.2.0",
"eslint-plugin-chai-friendly": "^0.6.0",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-mocha": "^8.0.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^5.0.0",
"grunt-bunyan": "~0.5.0",
"grunt-execute": "~0.1.5",
"grunt-available-tasks": "~0.4.1",
"grunt-concurrent": "~0.4.3",
"prettier": "^2.2.1",
"semver": "~2.2.1",
"knox": "~0.8.9"
}
}

116
server-ce/tasks/CreateAndDestroyUsers.js
View file

@ -1,116 +0,0 @@
/* eslint-disable
no-undef,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
module.exports = function (grunt) {
grunt.registerTask(
'user:create-admin',
'Create a user with the given email address and make them an admin. Update in place if the user already exists. Usage: grunt user:create-admin --email joe@example.com',
function () {
const done = this.async()
const email = grunt.option('email')
if (email == null) {
console.error('Usage: grunt user:create-admin --email=joe@example.com')
process.exit(1)
}
const settings = require('@overleaf/settings')
const mongodb = require('../web/app/src/infrastructure/mongodb')
const UserRegistrationHandler = require('../web/app/src/Features/User/UserRegistrationHandler')
const OneTimeTokenHandler = require('../web/app/src/Features/Security/OneTimeTokenHandler')
return mongodb.waitForDb().then(() =>
UserRegistrationHandler.registerNewUser(
{
email,
password: require('crypto').randomBytes(32).toString('hex'),
},
function (error, user) {
if (
error != null &&
(error != null ? error.message : undefined) !==
'EmailAlreadyRegistered'
) {
throw error
}
user.isAdmin = true
return user.save(function (error) {
if (error != null) {
throw error
}
const ONE_WEEK = 7 * 24 * 60 * 60 // seconds
return OneTimeTokenHandler.getNewToken(
'password',
{
expiresIn: ONE_WEEK,
email: user.email,
user_id: user._id.toString(),
},
function (err, token) {
if (err != null) {
return next(err)
}
console.log('')
console.log(`\
Successfully created ${email} as an admin user.
Please visit the following URL to set a password for ${email} and log in:
${settings.siteUrl}/user/password/set?passwordResetToken=${token}
\
`)
return done()
}
)
})
}
)
)
}
)
return grunt.registerTask(
'user:delete',
'deletes a user and all their data, Usage: grunt user:delete --email joe@example.com',
function () {
const done = this.async()
const email = grunt.option('email')
if (email == null) {
console.error('Usage: grunt user:delete --email=joe@example.com')
process.exit(1)
}
const settings = require('@overleaf/settings')
const mongodb = require('../web/app/src/infrastructure/mongodb')
const UserGetter = require('../web/app/src/Features/User/UserGetter')
const UserDeleter = require('../web/app/src/Features/User/UserDeleter')
return mongodb.waitForDb().then(() =>
UserGetter.getUser({ email }, function (error, user) {
if (error != null) {
throw error
}
if (user == null) {
console.log(
`user ${email} not in database, potentially already deleted`
)
return done()
}
return UserDeleter.deleteUser(user._id, function (err) {
if (err != null) {
throw err
}
return done()
})
})
)
}
)
}