Update default config to be more sane

server-ce/.gitignore

@@ -12,11 +12,8 @@ tags
 chat
 spelling
 
-compiles
-cache
-user_files
-template_files
 data
+tmp
 
 db.sqlite
 

server-ce/Gruntfile.coffee

@@ -5,6 +5,8 @@ rimraf = require "rimraf"
 Path = require "path"
 semver = require "semver"
 knox = require "knox"
+crypto = require "crypto"
+async = require "async"
 
 SERVICES = [{
 	name: "web"
@@ -75,7 +77,7 @@ module.exports = (grunt) ->
 						"Misc": [
 							"help"
 						]
-						"Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:config"])
+						"Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:dirs", "install:config"])
 						"Update tasks": ("update:#{service.name}" for service in SERVICES).concat(["update:all", "update"])
 						"Config tasks": ["install:config"]
 						"Checks": ["check", "check:redis", "check:latexmk", "check:s3", "check:make"]
@@ -92,6 +94,8 @@ module.exports = (grunt) ->
 
 	grunt.registerTask 'install:config', "Copy the example config into the real config", () ->
 		Helpers.installConfig @async()
+	grunt.registerTask 'install:dirs', "Copy the example config into the real config", () ->
+		Helpers.createDataDirs @async()
 	grunt.registerTask 'install:all', "Download and set up all ShareLaTeX services",
 		["check:make"].concat(
 			("install:#{service.name}" for service in SERVICES)
@@ -164,12 +168,34 @@ module.exports = (grunt) ->
 			proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir
 			proc.on "close", () ->
 				callback()
+				
+		createDataDirs: (callback = (error) ->) ->
+			DIRS = [
+				"tmp/dumpFolder"
+				"tmp/uploads"
+				"data/user_files"
+				"data/compiles"
+				"data/cache"
+			]
+			jobs = []
+			for dir in DIRS
+				do (dir) ->
+					jobs.push (callback) ->
+						path = Path.join(__dirname, dir)
+						grunt.log.writeln "Ensuring '#{path}' exists"
+						exec "mkdir -p #{path}", callback
+			async.series jobs, callback
 
 		installConfig: (callback = (error) ->) ->
-			if !fs.existsSync("config/settings.development.coffee")
+			src = "config/settings.development.coffee.example"
-				grunt.log.writeln "Copying example config into config/settings.development.coffee"
+			dest = "config/settings.development.coffee"
-				exec "cp config/settings.development.coffee.example config/settings.development.coffee", (error, stdout, stderr) ->
+			if !fs.existsSync(dest)
-					callback(error)
+				grunt.log.writeln "Creating config at #{dest}"
+				config = fs.readFileSync(src).toString()
+				config = config.replace /CRYPTO_RANDOM/g, () ->
+					crypto.randomBytes(64).toString("hex")
+				fs.writeFileSync dest, config
+				callback()
 			else
 				grunt.log.writeln "Config file already exists. Skipping."
 				callback()

server-ce/config/settings.development.coffee.example

@@ -1,281 +1,219 @@
 Path = require('path')
-http = require('http')
-http.globalAgent.maxSockets = 300
-
-# Make time interval config easier.
-seconds = 1000
-minutes = 60 * seconds
 
 # These credentials are used for authenticating api requests
 # between services that may need to go over public channels
 httpAuthUser = "sharelatex"
-httpAuthPass = "password"
+httpAuthPass = "CRYPTO_RANDOM" # Randomly generated for you
 httpAuthUsers = {}
 httpAuthUsers[httpAuthUser] = httpAuthPass
 
-sessionSecret = "secret-please-change"
+DATA_DIR = Path.resolve(Path.join(__dirname, "..", "data"))
+TMP_DIR = Path.resolve(Path.join(__dirname, "..", "tmp"))
 
 module.exports =
-	# File storage
-	# ------------
-	#
-	# ShareLaTeX needs somewhere to store binary files like images.
-	# There are currently two options:
-	#     Your local filesystem (the default)
-	#     Amazon S3
-	filestore:
-		# which backend persistor to use.
-		# choices are 
-		# s3 - Amazon S3
-		# fs - local filesystem
-		backend: "fs"	
-		stores:
-			# where to store user and template binary files
-			#
-			# For Amazon S3 this is the bucket name to store binary files
-			#
-			# For local filesystem this is the directory to store the files in.
-			# This path must exist, not be tmpfs and be writable to by the user sharelatex is run as.
-			user_files: Path.resolve(__dirname + "/../user_files")
-		# Uncomment if you need to configure your S3 credentials
-		# s3:
-		# 	# if you are using S3, then fill in your S3 details below
-		# 	key: ""
-		# 	secret: ""
-
 	# Databases
 	# ---------
+
+	# ShareLaTeX's main persistant data store is MongoDB (http://www.mongodb.org/)
+	# Documentation about the URL connection string format can be found at:
+	#
+	#    http://docs.mongodb.org/manual/reference/connection-string/
+	# 
+	# The following works out of the box with Mongo's default settings:
 	mongo:
 		url : 'mongodb://127.0.0.1/sharelatex'
 
+	# Redis is used in ShareLaTeX for high volume queries, like real-time
+	# editing, and session management.
+	#
+	# The following config will work with Redis's default settings:
 	redis:
 		web:
 			host: "localhost"
 			port: "6379"
 			password: ""
 
-		api:
+	# The compile server (the clsi) uses a SQL database to cache files and
-			host: "localhost"
+	# meta-data. sqllite is the default, and the load is low enough that this will
-			port: "6379"
+	# be fine in production (we use sqllite at sharelatex.com).
-			password: ""
+	#
-
+	# If you want to configure a different database, see the Sequelize documentation
-		fairy:
+	# for available options:
-			host: "localhost"
+	#
-			port: "6379"
+	#    https://github.com/sequelize/sequelize/wiki/API-Reference-Sequelize#example-usage
-			password: ""
+	#
-
 	mysql:
 		clsi:
 			database: "clsi"
 			username: "clsi"
 			password: ""
 			dialect: "sqlite"
-			storage: Path.resolve(__dirname + "/../db.sqlite")
+			storage: Path.join(DATA_DIR, "db.sqlite")
 
-	# Service locations
+	# File storage
-	# -----------------
+	# ------------
 
-	# Configure which ports to run each service on. Generally you
+	# ShareLaTeX can store binary files like images either locally or in Amazon
-	# can leave these as they are unless you have some other services
+	# S3. The default is locally:
-	# running which conflict, or want to run the web process on port 80.
+	filestore:
-	internal:
+		backend: "fs"	
-		web:
+		stores:
-			port: webPort = 3000
+			user_files: Path.join(DATA_DIR, "user_files")
-			host: "localhost"
+			
-		documentupdater:
+	# To use Amazon S3 as a storage backend, comment out the above config, and
-			port: docUpdaterPort = 3003
+	# uncomment the following, filling in your key, secret, and bucket name:
-			host: "localhost"
-		filestore:
-			port: filestorePort = 3009
-			host: "localhost"
-		chat:
-			port: chatPort = 3010
-			host: "localhost"
-		tags:
-			port: tagsPort = 3012
-			host: "localhost"
-		clsi:
-			port: clsiPort = 3013
-			host: "localhost"
-		trackchanges:
-			port: trackchangesPort = 3015
-			host: "localhost"
-		docstore:
-			port: docstorePort = 3016
-			host: "localhost"
-
-	# Tell each service where to find the other services. If everything
-	# is running locally then this is easy, but they exist as separate config
-	# options incase you want to run some services on remote hosts.
-	apis:
-		web:
-			url: "http://localhost:#{webPort}"
-			user: httpAuthUser
-			pass: httpAuthPass
-		documentupdater:
-			url : "http://localhost:#{docUpdaterPort}"
-		clsi:
-			url: "http://localhost:#{clsiPort}"
-		filestore:
-			url: "http://localhost:#{filestorePort}"
-		trackchanges:
-			url: "http://localhost:#{trackchangesPort}"
-		docstore:
-			url: "http://localhost:#{docstorePort}"
-		thirdPartyDataStore:
-			url : "http://localhost:3002"
-			emptyProjectFlushDelayMiliseconds: 5 * seconds
-		tags:
-			url :"http://localhost:#{tagsPort}"
-		spelling:
-			url : "http://localhost:3005"
-		versioning:
-			snapshotwaitms:3000
-			url: "http://localhost:4000"
-			username: httpAuthUser
-			password: httpAuthPass
-		recurly:
-			privateKey: ""
-			apiKey: ""
-			subdomain: ""
-		chat:
-			url: "http://localhost:#{chatPort}"
-		templates:
-			port: 3007
-		blog:
-			port: 3008
-		templates_api:
-			url: "http://localhost:3007"
-
-	# Where your instance of ShareLaTeX can be found publically. Used in emails
-	# that are sent out, generated links, etc.
-	siteUrl : 'http://localhost:3000'
-
-	# Same, but with http auth credentials.
-	httpAuthSiteUrl: 'http://#{httpAuthUser}:#{httpAuthPass}@localhost:3000'
-
-	# Security
-	# --------
-	security:
-		sessionSecret: sessionSecret
-
-	httpAuthUsers: httpAuthUsers
-
-	# Default features
-	# ----------------
 	#
-	# You can select the features that are enabled by default for new
+	# filestore:
-	# new users.
+	# 	backend: "s3"
-	defaultFeatures: defaultFeatures =
+	# 	stores:
-		collaborators: -1
+	# 		user_files: "BUCKET_NAME"
-		dropbox: true
+	# 	s3:
-		versioning: true
+	# 		key: "AWS_KEY"
+	# 		secret: "AWS_SECRET"
+	# 		
 
-	plans: plans = [{
+	# Local disk caching
-		planCode: "personal"
-		name: "Personal"
-		price: 0
-		features: defaultFeatures
-	}]
-
-	# Spelling languages
 	# ------------------
-	#
+	path:
-	# You must have the corresponding aspell package installed to 
+		# If we ever need to write something to disk (e.g. incoming requests
-	# be able to use a language.
+		# that need processing but may be too big for memory), then write
-	languages: [
+		# them to disk here:
-		{name: "English", code: "en"}
+		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")
-	]
+		# Where to write uploads before they are processed
+		uploadFolder: Path.join(TMP_DIR, "uploads")
+		# Where to write the project to disk before running LaTeX on it
+		compilesDir:  Path.join(DATA_DIR, "compiles")
+		# Where to cache downloaded URLs for the CLSI
+		clsiCacheDir: Path.join(DATA_DIR, "cache")
 
-	# Email support
+	# Server Config
 	# -------------
-	#
-	# ShareLaTeX uses nodemailer (http://www.nodemailer.com/) to send transactional emails.
-	# To see the range of transport and options they support, see http://www.nodemailer.com/docs/transports
-	# email:
-	#	fromAddress: ""
-	#	replyTo: ""
-	#	lifecycle: false
-	#	transport: "SES"
-	#	parameters:
-	#		AWSAccessKeyID: ""
-	#		AWSSecretKey: ""
 
+	# Where your instance of ShareLaTeX can be found publicly. This is used
+	# when emails are sent out and in generated links:
+	siteUrl : 'http://localhost:3000'
+	
+	# If provided, a sessionSecret is used to sign cookies so that they cannot be
+	# spoofed. This is recommended.
+	security:
+		sessionSecret: "CRYPTO_RANDOM" # This was randomly generated for you
 
-	# Third party services
+	# These credentials are used for authenticating api requests
-	# --------------------
+	# between services that may need to go over public channels
-	#
+	httpAuthUsers: httpAuthUsers
-	# ShareLaTeX's regular newsletter is managed by Markdown mail. Add your
+	
-	# credentials here to integrate with this.
-	# markdownmail:
-	# 	secret: ""
-	# 	list_id: ""
-	#
-	# Fill in your unique token from various analytics services to enable
-	# them.
-	# analytics:
-	# 	mixpanel:
-	# 		token: ""
-	# 	ga:
-	# 		token: ""
-	# 	heap:
-	# 		token: ""
-	# 
-	# ShareLaTeX's help desk is provided by tenderapp.com
-	# tenderUrl: ""
-	#
-
-	# Production Settings
-	# -------------------
-
 	# Should javascript assets be served minified or not. Note that you will
 	# need to run `grunt compile:minify` within the web-sharelatex directory
 	# to generate these.
 	useMinifiedJs: false
 
 	# Should static assets be sent with a header to tell the browser to cache
-	# them.
+	# them. This should be false in development where changes are being made,
+	# but should be set to true in production.
 	cacheStaticAssets: false
 
 	# If you are running ShareLaTeX over https, set this to true to send the
 	# cookie with a secure flag (recommended).
 	secureCookie: false
-
-	# Internal configs
-	# ----------------
-	path:
-		# If we ever need to write something to disk (e.g. incoming requests
-		# that need processing but may be too big for memory, then write
-		# them to disk here).
-		dumpFolder: Path.resolve "data/dumpFolder"
-		# Where to write the project to disk before running LaTeX on it
-		compilesDir:  Path.resolve(__dirname + "/../compiles")
-		# Where to cache downloaded URLs for the CLSI
-		clsiCacheDir: Path.resolve(__dirname + "/../cache")
 	
-	# Automatic Snapshots
+	# If you are running ShareLaTeX behind a proxy (like Apache, Nginx, etc)
-	# -------------------
+	# then set this to true to allow it to correctly detect the forwarded IP
-	automaticSnapshots:
+	# address and http/https protocol information.
-		# How long should we wait after the user last edited to 
+	behindProxy: false
-		# take a snapshot?
-		waitTimeAfterLastEdit: 5 * minutes
-		# Even if edits are still taking place, this is maximum
-		# time to wait before taking another snapshot.
-		maxTimeBetweenSnapshots: 30 * minutes
 
-	# Smoke test
+	# Sending Email
-	# ----------
+	# -------------
-	# Provide log in credentials and a project to be able to run
-	# some basic smoke tests to check the core functionality.
 	#
-	# smokeTest:
+	# You must configure a mail server to be able to send invite emails from
-	# 	user: ""
+	# ShareLaTeX. The config settings are passed to nodemailer. See the nodemailer
-	# 	password: ""
+	# documentation for available options:
-	# 	projectId: ""
+	#
+	#     http://www.nodemailer.com/docs/transports
+	#
+	# email:
+	#	fromAddress: ""
+	#	replyTo: ""
+	#	transport: "SES"
+	#	parameters:
+	#		AWSAccessKeyID: ""
+	#		AWSSecretKey: ""
 
-	# Filestore health check
+	# Spell Check Languages
-	# ----------------------
+	# ---------------------
-	# Project and file details to check in filestore when calling /health_check
+	#
-	# health_check:
+	# You must have the corresponding aspell dictionary installed to 
-	# 	project_id: ""
+	# be able to use a language. Run `grunt check:aspell` to check which
-	# 	file_id: ""
+	# dictionaries you have installed. These should be set for the `code` for
+	# each language.
+	languages: [
+		{name: "English", code: "en"}
+	]
+	
+	# Service locations
+	# -----------------
+
+	# ShareLaTeX is comprised of many small services, which each expose
+	# an HTTP API running on a different port. Generally you
+	# can leave these as they are unless you have some other services
+	# running which conflict, or want to run the web process on port 80.
+	# internal:
+	# 	web:
+	# 		port: webPort = 3000
+	# 		host: "localhost"
+	# 	documentupdater:
+	# 		port: docUpdaterPort = 3003
+	# 		host: "localhost"
+	# 	filestore:
+	# 		port: filestorePort = 3009
+	# 		host: "localhost"
+	# 	chat:
+	# 		port: chatPort = 3010
+	# 		host: "localhost"
+	# 	tags:
+	# 		port: tagsPort = 3012
+	# 		host: "localhost"
+	# 	clsi:
+	# 		port: clsiPort = 3013
+	# 		host: "localhost"
+	# 	trackchanges:
+	# 		port: trackchangesPort = 3015
+	# 		host: "localhost"
+	# 	docstore:
+	# 		port: docstorePort = 3016
+	# 		host: "localhost"
+	# 	spelling:
+	# 		port: spellingPort = 3005
+	# 		host: "localhost"
+
+	# If you change the above config, or run some services on remote servers,
+	# you need to tell the other services where to find them:
+	apis:
+		web:
+			url: "http://localhost:3000"
+			user: httpAuthUser
+			pass: httpAuthPass
+	# 	documentupdater:
+	# 		url : "http://localhost:#{docUpdaterPort}"
+	# 	clsi:
+	# 		url: "http://localhost:#{clsiPort}"
+	# 	filestore:
+	# 		url: "http://localhost:#{filestorePort}"
+	# 	trackchanges:
+	# 		url: "http://localhost:#{trackchangesPort}"
+	# 	docstore:
+	# 		url: "http://localhost:#{docstorePort}"
+	# 	tags:
+	# 		url: "http://localhost:#{tagsPort}"
+	# 	spelling:
+	# 		url: "http://localhost:#{spellingPort}"
+	# 	chat:
+	# 		url: "http://localhost:#{chatPort}"
+	
+
+# With lots of incoming and outgoing HTTP connections to different services,
+# sometimes long running, it is a good idea to increase the default number
+# of sockets that Node will hold open.
+http = require('http')
+http.globalAgent.maxSockets = 300
+https = require('https')
+https.globalAgent.maxSockets = 300

server-ce/package.json

@@ -3,6 +3,7 @@
   "version": "0.0.1",
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
+    "async": "^0.9.0",
     "rimraf": "~2.2.6",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
   },