overleaf/server-ce/Gruntfile.coffee

545 lines
18 KiB
CoffeeScript
Raw Normal View History

fs = require "fs"
spawn = require("child_process").spawn
exec = require("child_process").exec
rimraf = require "rimraf"
2014-02-12 07:15:47 -05:00
Path = require "path"
semver = require "semver"
2014-02-22 10:02:21 -05:00
knox = require "knox"
2014-08-15 10:05:07 -04:00
crypto = require "crypto"
async = require "async"
2014-02-12 06:18:25 -05:00
SERVICES = [{
name: "web"
repo: "https://github.com/sharelatex/web-sharelatex.git"
version: "master"
2014-11-21 07:44:53 -05:00
}, {
name: "real-time"
repo: "https://github.com/sharelatex/real-time-sharelatex.git"
version: "master"
2014-02-12 06:18:25 -05:00
}, {
name: "document-updater"
repo: "https://github.com/sharelatex/document-updater-sharelatex.git"
version: "master"
2014-02-13 07:37:47 -05:00
}, {
name: "clsi"
repo: "https://github.com/sharelatex/clsi-sharelatex.git"
version: "master"
2014-02-14 12:30:43 -05:00
}, {
name: "filestore"
repo: "https://github.com/sharelatex/filestore-sharelatex.git"
version: "master"
}, {
name: "track-changes"
repo: "https://github.com/sharelatex/track-changes-sharelatex.git"
version: "master"
2014-04-29 07:05:18 -04:00
}, {
name: "docstore"
repo: "https://github.com/sharelatex/docstore-sharelatex.git"
version: "master"
2014-08-15 07:00:39 -04:00
}, {
name: "chat"
repo: "https://github.com/sharelatex/chat-sharelatex.git"
version: "master"
2014-08-15 07:00:39 -04:00
}, {
name: "tags"
repo: "https://github.com/sharelatex/tags-sharelatex.git"
version: "master"
2014-08-15 07:39:55 -04:00
}, {
name: "spelling"
repo: "https://github.com/sharelatex/spelling-sharelatex.git"
version: "master"
}]
module.exports = (grunt) ->
grunt.loadNpmTasks 'grunt-bunyan'
grunt.loadNpmTasks 'grunt-execute'
grunt.loadNpmTasks 'grunt-available-tasks'
grunt.loadNpmTasks 'grunt-concurrent'
2015-01-28 12:50:39 -05:00
grunt.loadNpmTasks "grunt-contrib-coffee"
2014-02-13 07:37:47 -05:00
execute = {}
for service in SERVICES
2014-02-13 07:37:47 -05:00
execute[service.name] =
src: "#{service.name}/app.js"
grunt.initConfig
2014-02-13 07:37:47 -05:00
execute: execute
concurrent:
all:
2014-02-13 07:37:47 -05:00
tasks: ("run:#{service.name}" for service in SERVICES)
options:
2014-02-13 07:37:47 -05:00
limit: SERVICES.length
logConcurrentOutput: true
2015-01-28 12:50:39 -05:00
coffee:
migrate:
expand: true,
flatten: false,
cwd: './',
src: ['./migrations/*.coffee'],
dest: './',
ext: '.js'
options:
bare:true
availabletasks:
tasks:
options:
filter: 'exclude',
tasks: [
'concurrent'
'execute'
'bunyan'
'availabletasks'
]
groups:
"Run tasks": [
"run"
"run:all"
"default"
2014-02-13 07:37:47 -05:00
].concat ("run:#{service.name}" for service in SERVICES)
"Misc": [
"help"
]
2014-08-15 10:05:07 -04:00
"Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:dirs", "install:config"])
"Update tasks": ("update:#{service.name}" for service in SERVICES).concat(["update:all", "update"])
"Config tasks": ["install:config"]
"Checks": ["check", "check:redis", "check:latexmk", "check:s3", "check:make"]
for service in SERVICES
2014-02-12 06:18:25 -05:00
do (service) ->
grunt.registerTask "install:#{service.name}", "Download and set up the #{service.name} service", () ->
done = @async()
Helpers.installService(service, done)
2014-02-12 06:18:25 -05:00
grunt.registerTask "update:#{service.name}", "Checkout and update the #{service.name} service", () ->
done = @async()
Helpers.updateService(service, done)
2014-02-12 07:15:47 -05:00
grunt.registerTask "run:#{service.name}", "Run the ShareLaTeX #{service.name} service", ["bunyan", "execute:#{service.name}"]
2014-08-19 09:46:45 -04:00
grunt.registerTask "release:#{service.name}", "Create a new release version of #{service.name} (specify with --release option)", () ->
done = @async()
Helpers.createNewRelease(service, grunt.option("release"), done)
2014-02-12 06:18:25 -05:00
grunt.registerTask 'install:config', "Copy the example config into the real config", () ->
Helpers.installConfig @async()
2014-08-15 10:05:07 -04:00
grunt.registerTask 'install:dirs', "Copy the example config into the real config", () ->
Helpers.createDataDirs @async()
grunt.registerTask 'install:all', "Download and set up all ShareLaTeX services",
["check:make"].concat(
("install:#{service.name}" for service in SERVICES)
).concat(["install:config", "install:dirs"])
2014-02-12 06:24:13 -05:00
grunt.registerTask 'install', 'install:all'
grunt.registerTask 'update:all', "Checkout and update all ShareLaTeX services",
["check:make"].concat(
("update:#{service.name}" for service in SERVICES)
)
2014-02-12 06:24:13 -05:00
grunt.registerTask 'update', 'update:all'
grunt.registerTask 'run', "Run all of the sharelatex processes", ['concurrent:all']
2014-02-08 16:52:45 -05:00
grunt.registerTask 'run:all', 'run'
2014-02-12 06:24:13 -05:00
grunt.registerTask 'help', 'Display this help list', 'availabletasks'
grunt.registerTask 'default', 'run'
grunt.registerTask "check:redis", "Check that redis is installed and running", () ->
Helpers.checkRedis @async()
grunt.registerTask "check:latexmk", "Check that latexmk is installed", () ->
Helpers.checkLatexmk @async()
2014-02-22 10:02:21 -05:00
grunt.registerTask "check:s3", "Check that Amazon S3 credentials are configured", () ->
Helpers.checkS3 @async()
2014-03-04 09:35:49 -05:00
grunt.registerTask "check:fs", "Check that local filesystem options are configured", () ->
Helpers.checkFS @async()
2014-08-15 07:57:06 -04:00
grunt.registerTask "check:aspell", "Check that aspell is installed", () ->
Helpers.checkAspell @async()
grunt.registerTask "check:make", "Check that make is installed", () ->
Helpers.checkMake @async()
2014-08-15 07:57:06 -04:00
grunt.registerTask "check", "Check that you have the required dependencies installed", ["check:redis", "check:latexmk", "check:s3", "check:fs", "check:aspell"]
2014-08-18 05:46:42 -04:00
grunt.registerTask "build:deb", "Build an installable .deb file from the current directory", () ->
Helpers.buildDeb @async()
2014-08-18 05:46:42 -04:00
grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
Helpers.buildUpstartScripts()
2015-01-28 12:50:39 -05:00
grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
Helpers =
installService: (service, callback = (error) ->) ->
Helpers.cloneGitRepo service, (error) ->
return callback(error) if error?
Helpers.installNpmModules service, (error) ->
return callback(error) if error?
Helpers.rebuildNpmModules service, (error) ->
return callback(error) if error?
Helpers.runGruntInstall service, (error) ->
return callback(error) if error?
callback()
updateService: (service, callback = (error) ->) ->
Helpers.updateGitRepo service, (error) ->
return callback(error) if error?
Helpers.installNpmModules service, (error) ->
return callback(error) if error?
Helpers.rebuildNpmModules service, (error) ->
return callback(error) if error?
Helpers.runGruntInstall service, (error) ->
return callback(error) if error?
callback()
cloneGitRepo: (service, callback = (error) ->) ->
repo_src = service.repo
dir = service.name
if !fs.existsSync(dir)
proc = spawn "git", [
"clone",
"-b", service.version,
repo_src,
dir
], stdio: "inherit"
proc.on "close", () ->
callback()
else
console.log "#{dir} already installed, skipping."
callback()
updateGitRepo: (service, callback = (error) ->) ->
dir = service.name
proc = spawn "git", ["checkout", service.version], cwd: dir, stdio: "inherit"
proc.on "close", () ->
proc = spawn "git", ["pull"], cwd: dir, stdio: "inherit"
proc.on "close", () ->
callback()
2014-08-19 09:46:45 -04:00
createNewRelease: (service, version, callback = (error) ->) ->
dir = service.name
proc = spawn "sed", [
"-i", "",
"s/\"version\".*$/\"version\": \"#{version}\",/g",
"package.json"
], cwd: dir, stdio: "inherit"
proc.on "close", () ->
proc = spawn "git", ["commit", "-a", "-m", "Release version #{version}"], cwd: dir, stdio: "inherit"
proc.on "close", () ->
proc = spawn "git", ["tag", "v#{version}"], cwd: dir, stdio: "inherit"
proc.on "close", () ->
proc = spawn "git", ["push"], cwd: dir, stdio: "inherit"
proc.on "close", () ->
proc = spawn "git", ["push", "--tags"], cwd: dir, stdio: "inherit"
proc.on "close", () ->
callback()
installNpmModules: (service, callback = (error) ->) ->
dir = service.name
proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir
proc.on "close", () ->
callback()
# work around for https://github.com/npm/npm/issues/5400
# where binary modules are not built due to bug in npm
rebuildNpmModules: (service, callback = (error) ->) ->
dir = service.name
proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir
proc.on "close", () ->
callback()
2014-08-15 10:05:07 -04:00
createDataDirs: (callback = (error) ->) ->
DIRS = [
"tmp/dumpFolder"
"tmp/uploads"
"data/user_files"
"data/compiles"
"data/cache"
]
jobs = []
for dir in DIRS
do (dir) ->
jobs.push (callback) ->
path = Path.join(__dirname, dir)
grunt.log.writeln "Ensuring '#{path}' exists"
exec "mkdir -p #{path}", callback
async.series jobs, callback
installConfig: (callback = (error) ->) ->
2014-08-15 10:05:07 -04:00
src = "config/settings.development.coffee.example"
dest = "config/settings.development.coffee"
if !fs.existsSync(dest)
grunt.log.writeln "Creating config at #{dest}"
config = fs.readFileSync(src).toString()
config = config.replace /CRYPTO_RANDOM/g, () ->
crypto.randomBytes(64).toString("hex")
fs.writeFileSync dest, config
callback()
else
grunt.log.writeln "Config file already exists. Skipping."
callback()
runGruntInstall: (service, callback = (error) ->) ->
dir = service.name
proc = spawn "grunt", ["install"], stdio: "inherit", cwd: dir
proc.on "close", () ->
callback()
checkRedis: (callback = (error) ->) ->
grunt.log.write "Checking Redis is running... "
exec "redis-cli info", (error, stdout, stderr) ->
if error? and error.message.match("Could not connect")
grunt.log.error "FAIL. Redis is not running"
2014-02-22 10:02:21 -05:00
return callback(error)
else if error?
return callback(error)
else
m = stdout.match(/redis_version:(.*)/)
if !m?
grunt.log.error "FAIL."
grunt.log.error "Unknown redis version"
2014-02-22 10:02:21 -05:00
error = new Error("Unknown redis version")
else
version = m[1]
2014-04-09 05:56:00 -04:00
if semver.gte(version, "2.6.12")
grunt.log.writeln "OK."
grunt.log.writeln "Running Redis version #{version}"
else
grunt.log.error "FAIL."
2014-04-09 05:56:00 -04:00
grunt.log.error "Redis version is too old (#{version}). Must be 2.6.12 or greater."
error = new Error("Redis version is too old (#{version}). Must be 2.6.12 or greater.")
2014-02-22 10:02:21 -05:00
callback(error)
checkLatexmk: (callback = (error) ->) ->
grunt.log.write "Checking latexmk is installed... "
2014-02-22 09:08:49 -05:00
exec "latexmk --version", (error, stdout, stderr) ->
2014-08-15 08:34:57 -04:00
if error? and error.message.match("not found")
grunt.log.error "FAIL."
grunt.log.errorlns """
Either latexmk is not installed or is not in your PATH.
latexmk comes with TexLive 2013, and must be a version from 2013 or later.
2014-08-15 08:44:21 -04:00
If you have already have TeXLive installed, then make sure it is
included in your PATH (example for 64-bit linux):
export PATH=$PATH:/usr/local/texlive/2014/bin/x86_64-linux/
This is a not a fatal error, but compiling will not work without latexmk.
"""
2014-02-22 10:02:21 -05:00
return callback(error)
else if error?
return callback(error)
else
m = stdout.match(/Version (.*)/)
if !m?
grunt.log.error "FAIL."
grunt.log.error "Unknown latexmk version"
2014-02-22 10:02:21 -05:00
error = new Error("Unknown latexmk version")
else
version = m[1]
2014-02-22 09:08:49 -05:00
if semver.gte(version + ".0", "4.39.0")
grunt.log.writeln "OK."
grunt.log.writeln "Running latexmk version #{version}"
else
grunt.log.error "FAIL."
grunt.log.errorlns """
latexmk version is too old (#{version}). Must be 4.39 or greater.
This is a not a fatal error, but compiling will not work without latexmk
"""
2014-02-22 10:02:21 -05:00
error = new Error("latexmk is too old")
callback(error)
2014-08-15 07:57:06 -04:00
checkAspell: (callback = (error) ->) ->
grunt.log.write "Checking aspell is installed... "
exec "aspell dump dicts", (error, stdout, stderr) ->
2014-08-15 08:34:57 -04:00
if error? and error.message.match("not found")
2014-08-15 07:57:06 -04:00
grunt.log.error "FAIL."
grunt.log.errorlns """
Either aspell is not installed or is not in your PATH.
On Ubuntu you can install aspell with:
2014-08-15 08:36:32 -04:00
2014-08-15 07:57:06 -04:00
sudo apt-get install aspell
Or on a mac:
2014-08-15 08:36:32 -04:00
2014-08-15 07:57:06 -04:00
brew install aspell
2014-08-15 08:44:21 -04:00
This is not a fatal error, but the spell-checker will not work without aspell
2014-08-15 07:57:06 -04:00
"""
return callback(error)
else if error?
return callback(error)
else
grunt.log.writeln "OK."
grunt.log.writeln "The following spell check dictionaries are available:"
grunt.log.write stdout
callback()
callback(error)
2014-02-22 10:02:21 -05:00
checkS3: (callback = (error) ->) ->
2014-02-24 14:08:08 -05:00
Settings = require "settings-sharelatex"
2014-03-04 09:35:49 -05:00
if Settings.filestore.backend==""
grunt.log.writeln "No backend specified. Assuming Amazon S3"
Settings.filestore.backend = "s3"
if Settings.filestore.backend=="s3"
grunt.log.write "Checking S3 credentials... "
try
client = knox.createClient({
key: Settings.filestore.s3.key
secret: Settings.filestore.s3.secret
bucket: Settings.filestore.stores.user_files
})
catch e
2014-02-22 10:02:21 -05:00
grunt.log.error "FAIL."
grunt.log.errorlns """
2014-03-05 07:52:04 -05:00
Please configure your Amazon S3 credentials in config/settings.development.coffee
2014-03-04 09:35:49 -05:00
Amazon S3 (Simple Storage Service) is a cloud storage service provided by
Amazon. ShareLaTeX uses S3 for storing binary files like images. You can
sign up for an account and find out more at:
http://aws.amazon.com/s3/
2014-02-22 10:02:21 -05:00
"""
2014-03-04 09:35:49 -05:00
return callback()
client.getFile "does-not-exist", (error, response) ->
unless response? and response.statusCode == 404
grunt.log.error "FAIL."
grunt.log.errorlns """
Could not connect to Amazon S3. Please check your credentials.
"""
else
2014-08-15 07:57:06 -04:00
grunt.log.writeln "OK."
2014-03-04 09:35:49 -05:00
callback()
else
grunt.log.writeln "Filestore other than S3 configured. Not checking S3."
callback()
2014-03-04 09:35:49 -05:00
checkFS: (callback = (error) ->) ->
Settings = require "settings-sharelatex"
if Settings.filestore.backend=="fs"
2014-08-15 08:45:26 -04:00
grunt.log.write "Checking FS configuration... "
2014-03-04 09:35:49 -05:00
fs = require("fs")
fs.exists Settings.filestore.stores.user_files, (exists) ->
if exists
2014-08-15 08:45:26 -04:00
grunt.log.writeln "OK."
2014-03-04 09:35:49 -05:00
else
grunt.log.error "FAIL."
grunt.log.errorlns """
Could not find directory "#{Settings.filestore.stores.user_files}".
Please check your configuration.
"""
2014-08-15 08:46:06 -04:00
callback()
2014-03-04 09:35:49 -05:00
else
grunt.log.writeln "Filestore other than FS configured. Not checking FS."
2014-08-15 08:46:06 -04:00
callback()
2014-03-04 09:35:49 -05:00
checkMake: (callback = (error) ->) ->
grunt.log.write "Checking make is installed... "
exec "make --version", (error, stdout, stderr) ->
2014-08-15 08:34:57 -04:00
if error? and error.message.match("not found")
grunt.log.error "FAIL."
grunt.log.errorlns """
Either make is not installed or is not in your path.
2014-08-15 08:36:32 -04:00
On Ubuntu you can install make with:
2014-08-15 08:36:32 -04:00
sudo apt-get install build-essential
2014-08-15 08:36:32 -04:00
"""
return callback(error)
else if error?
return callback(error)
else
grunt.log.write "OK."
return callback()
2014-08-18 05:46:42 -04:00
buildUpstartScripts: () ->
template = fs.readFileSync("package/upstart/sharelatex-template").toString()
for service in SERVICES
fs.writeFileSync "package/upstart/sharelatex-#{service.name}", template.replace(/__SERVICE__/g, service.name)
2014-08-18 10:13:48 -04:00
buildPackageSettingsFile: () ->
config = fs.readFileSync("config/settings.development.coffee.example").toString()
config = config.replace /DATA_DIR.*/, "DATA_DIR = '/var/lib/sharelatex/data'"
config = config.replace /TMP_DIR.*/, "TMP_DIR = '/var/lib/sharelatex/tmp'"
fs.writeFileSync "package/config/settings.coffee", config
buildDeb: (callback = (error) ->) ->
2014-05-28 07:28:43 -04:00
command = ["-s", "dir", "-t", "deb", "-n", "sharelatex", "-v", "0.0.1", "--verbose"]
command.push(
2014-05-28 07:28:43 -04:00
"--maintainer", "ShareLaTeX <team@sharelatex.com>"
2014-08-19 05:41:04 -04:00
"--config-files", "/etc/sharelatex/settings.coffee"
2014-08-19 07:28:19 -04:00
"--config-files", "/etc/nginx/conf.d/sharelatex.conf"
2014-08-18 05:46:42 -04:00
"--directories", "/var/lib/sharelatex"
"--directories", "/var/log/sharelatex"
)
command.push(
2014-05-28 07:28:43 -04:00
"--depends", "redis-server > 2.6.12"
2014-08-18 10:13:48 -04:00
"--depends", "mongodb-org > 2.4.0"
2014-05-28 07:28:43 -04:00
"--depends", "nodejs > 0.10.0"
)
2014-08-18 10:13:48 -04:00
@buildPackageSettingsFile()
2014-08-18 05:46:42 -04:00
@buildUpstartScripts()
for service in SERVICES
command.push(
"--deb-upstart", "package/upstart/sharelatex-#{service.name}"
)
after_install_script = """
#!/bin/sh
# Create random secret keys (twice, once for http auth pass, once for cookie secret).
sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
2014-08-18 10:13:48 -04:00
sudo adduser --system --group --home /var/www/sharelatex --no-create-home sharelatex
mkdir -p /var/log/sharelatex
chown sharelatex:sharelatex /var/log/sharelatex
2014-08-18 10:13:48 -04:00
mkdir -p /var/lib/sharelatex
"""
2014-08-18 10:13:48 -04:00
for dir in ["data/user_files", "tmp/uploads", "data/compiles", "data/cache", "tmp/dumpFolder"]
after_install_script += """
2014-08-18 05:46:42 -04:00
mkdir -p /var/lib/sharelatex/#{dir}
"""
2014-08-19 07:05:22 -04:00
after_install_script += """
chown -R sharelatex:sharelatex /var/lib/sharelatex
2014-08-19 07:05:22 -04:00
"""
for service in SERVICES
after_install_script += "service sharelatex-#{service.name} restart\n"
fs.writeFileSync "package/scripts/after_install.sh", after_install_script
command.push("--after-install", "package/scripts/after_install.sh")
command.push("--exclude", "**/.git")
command.push("--exclude", "**/node_modules/grunt-*")
for path in ["filestore/user_files", "filestore/uploads", "clsi/cache", "clsi/compiles"]
command.push "--exclude", path
for service in SERVICES
command.push "#{service.name}=/var/www/sharelatex/"
command.push(
"package/config/settings.coffee=/etc/sharelatex/settings.coffee"
2014-08-19 07:28:19 -04:00
"package/nginx/sharelatex=/etc/nginx/conf.d/sharelatex.conf"
)
2014-05-28 07:28:43 -04:00
console.log "fpm " + command.join(" ")
proc = spawn "fpm", command, stdio: "inherit"
2014-05-28 07:42:49 -04:00
proc.on "close", (code) ->
if code != 0
callback(new Error("exit code: #{code}"))
else
callback()