diff --git a/server-ce/.gitignore b/server-ce/.gitignore index de71076dff..8dd120550d 100644 --- a/server-ce/.gitignore +++ b/server-ce/.gitignore @@ -12,11 +12,8 @@ tags chat spelling -compiles -cache -user_files -template_files data +tmp db.sqlite diff --git a/server-ce/Gruntfile.coffee b/server-ce/Gruntfile.coffee index 1975c3f22a..f676cfbad1 100644 --- a/server-ce/Gruntfile.coffee +++ b/server-ce/Gruntfile.coffee @@ -5,6 +5,8 @@ rimraf = require "rimraf" Path = require "path" semver = require "semver" knox = require "knox" +crypto = require "crypto" +async = require "async" SERVICES = [{ name: "web" @@ -75,7 +77,7 @@ module.exports = (grunt) -> "Misc": [ "help" ] - "Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:config"]) + "Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:dirs", "install:config"]) "Update tasks": ("update:#{service.name}" for service in SERVICES).concat(["update:all", "update"]) "Config tasks": ["install:config"] "Checks": ["check", "check:redis", "check:latexmk", "check:s3", "check:make"] @@ -92,6 +94,8 @@ module.exports = (grunt) -> grunt.registerTask 'install:config', "Copy the example config into the real config", () -> Helpers.installConfig @async() + grunt.registerTask 'install:dirs', "Copy the example config into the real config", () -> + Helpers.createDataDirs @async() grunt.registerTask 'install:all', "Download and set up all ShareLaTeX services", ["check:make"].concat( ("install:#{service.name}" for service in SERVICES) @@ -164,12 +168,34 @@ module.exports = (grunt) -> proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir proc.on "close", () -> callback() + + createDataDirs: (callback = (error) ->) -> + DIRS = [ + "tmp/dumpFolder" + "tmp/uploads" + "data/user_files" + "data/compiles" + "data/cache" + ] + jobs = [] + for dir in DIRS + do (dir) -> + jobs.push (callback) -> + path = Path.join(__dirname, dir) + grunt.log.writeln "Ensuring '#{path}' exists" + exec "mkdir -p #{path}", callback + async.series jobs, callback installConfig: (callback = (error) ->) -> - if !fs.existsSync("config/settings.development.coffee") - grunt.log.writeln "Copying example config into config/settings.development.coffee" - exec "cp config/settings.development.coffee.example config/settings.development.coffee", (error, stdout, stderr) -> - callback(error) + src = "config/settings.development.coffee.example" + dest = "config/settings.development.coffee" + if !fs.existsSync(dest) + grunt.log.writeln "Creating config at #{dest}" + config = fs.readFileSync(src).toString() + config = config.replace /CRYPTO_RANDOM/g, () -> + crypto.randomBytes(64).toString("hex") + fs.writeFileSync dest, config + callback() else grunt.log.writeln "Config file already exists. Skipping." callback() diff --git a/server-ce/config/settings.development.coffee.example b/server-ce/config/settings.development.coffee.example index 91d698ae9a..c51e2be595 100644 --- a/server-ce/config/settings.development.coffee.example +++ b/server-ce/config/settings.development.coffee.example @@ -1,281 +1,219 @@ Path = require('path') -http = require('http') -http.globalAgent.maxSockets = 300 - -# Make time interval config easier. -seconds = 1000 -minutes = 60 * seconds # These credentials are used for authenticating api requests # between services that may need to go over public channels httpAuthUser = "sharelatex" -httpAuthPass = "password" +httpAuthPass = "CRYPTO_RANDOM" # Randomly generated for you httpAuthUsers = {} httpAuthUsers[httpAuthUser] = httpAuthPass -sessionSecret = "secret-please-change" +DATA_DIR = Path.resolve(Path.join(__dirname, "..", "data")) +TMP_DIR = Path.resolve(Path.join(__dirname, "..", "tmp")) module.exports = - # File storage - # ------------ - # - # ShareLaTeX needs somewhere to store binary files like images. - # There are currently two options: - # Your local filesystem (the default) - # Amazon S3 - filestore: - # which backend persistor to use. - # choices are - # s3 - Amazon S3 - # fs - local filesystem - backend: "fs" - stores: - # where to store user and template binary files - # - # For Amazon S3 this is the bucket name to store binary files - # - # For local filesystem this is the directory to store the files in. - # This path must exist, not be tmpfs and be writable to by the user sharelatex is run as. - user_files: Path.resolve(__dirname + "/../user_files") - # Uncomment if you need to configure your S3 credentials - # s3: - # # if you are using S3, then fill in your S3 details below - # key: "" - # secret: "" - # Databases # --------- + + # ShareLaTeX's main persistant data store is MongoDB (http://www.mongodb.org/) + # Documentation about the URL connection string format can be found at: + # + # http://docs.mongodb.org/manual/reference/connection-string/ + # + # The following works out of the box with Mongo's default settings: mongo: url : 'mongodb://127.0.0.1/sharelatex' + # Redis is used in ShareLaTeX for high volume queries, like real-time + # editing, and session management. + # + # The following config will work with Redis's default settings: redis: web: host: "localhost" port: "6379" password: "" - api: - host: "localhost" - port: "6379" - password: "" - - fairy: - host: "localhost" - port: "6379" - password: "" - + # The compile server (the clsi) uses a SQL database to cache files and + # meta-data. sqllite is the default, and the load is low enough that this will + # be fine in production (we use sqllite at sharelatex.com). + # + # If you want to configure a different database, see the Sequelize documentation + # for available options: + # + # https://github.com/sequelize/sequelize/wiki/API-Reference-Sequelize#example-usage + # mysql: clsi: database: "clsi" username: "clsi" password: "" dialect: "sqlite" - storage: Path.resolve(__dirname + "/../db.sqlite") + storage: Path.join(DATA_DIR, "db.sqlite") - # Service locations - # ----------------- + # File storage + # ------------ - # Configure which ports to run each service on. Generally you - # can leave these as they are unless you have some other services - # running which conflict, or want to run the web process on port 80. - internal: - web: - port: webPort = 3000 - host: "localhost" - documentupdater: - port: docUpdaterPort = 3003 - host: "localhost" - filestore: - port: filestorePort = 3009 - host: "localhost" - chat: - port: chatPort = 3010 - host: "localhost" - tags: - port: tagsPort = 3012 - host: "localhost" - clsi: - port: clsiPort = 3013 - host: "localhost" - trackchanges: - port: trackchangesPort = 3015 - host: "localhost" - docstore: - port: docstorePort = 3016 - host: "localhost" - - # Tell each service where to find the other services. If everything - # is running locally then this is easy, but they exist as separate config - # options incase you want to run some services on remote hosts. - apis: - web: - url: "http://localhost:#{webPort}" - user: httpAuthUser - pass: httpAuthPass - documentupdater: - url : "http://localhost:#{docUpdaterPort}" - clsi: - url: "http://localhost:#{clsiPort}" - filestore: - url: "http://localhost:#{filestorePort}" - trackchanges: - url: "http://localhost:#{trackchangesPort}" - docstore: - url: "http://localhost:#{docstorePort}" - thirdPartyDataStore: - url : "http://localhost:3002" - emptyProjectFlushDelayMiliseconds: 5 * seconds - tags: - url :"http://localhost:#{tagsPort}" - spelling: - url : "http://localhost:3005" - versioning: - snapshotwaitms:3000 - url: "http://localhost:4000" - username: httpAuthUser - password: httpAuthPass - recurly: - privateKey: "" - apiKey: "" - subdomain: "" - chat: - url: "http://localhost:#{chatPort}" - templates: - port: 3007 - blog: - port: 3008 - templates_api: - url: "http://localhost:3007" - - # Where your instance of ShareLaTeX can be found publically. Used in emails - # that are sent out, generated links, etc. - siteUrl : 'http://localhost:3000' - - # Same, but with http auth credentials. - httpAuthSiteUrl: 'http://#{httpAuthUser}:#{httpAuthPass}@localhost:3000' - - # Security - # -------- - security: - sessionSecret: sessionSecret - - httpAuthUsers: httpAuthUsers - - # Default features - # ---------------- + # ShareLaTeX can store binary files like images either locally or in Amazon + # S3. The default is locally: + filestore: + backend: "fs" + stores: + user_files: Path.join(DATA_DIR, "user_files") + + # To use Amazon S3 as a storage backend, comment out the above config, and + # uncomment the following, filling in your key, secret, and bucket name: # - # You can select the features that are enabled by default for new - # new users. - defaultFeatures: defaultFeatures = - collaborators: -1 - dropbox: true - versioning: true + # filestore: + # backend: "s3" + # stores: + # user_files: "BUCKET_NAME" + # s3: + # key: "AWS_KEY" + # secret: "AWS_SECRET" + # - plans: plans = [{ - planCode: "personal" - name: "Personal" - price: 0 - features: defaultFeatures - }] - - # Spelling languages + # Local disk caching # ------------------ - # - # You must have the corresponding aspell package installed to - # be able to use a language. - languages: [ - {name: "English", code: "en"} - ] + path: + # If we ever need to write something to disk (e.g. incoming requests + # that need processing but may be too big for memory), then write + # them to disk here: + dumpFolder: Path.join(TMP_DIR, "dumpFolder") + # Where to write uploads before they are processed + uploadFolder: Path.join(TMP_DIR, "uploads") + # Where to write the project to disk before running LaTeX on it + compilesDir: Path.join(DATA_DIR, "compiles") + # Where to cache downloaded URLs for the CLSI + clsiCacheDir: Path.join(DATA_DIR, "cache") - # Email support + # Server Config # ------------- - # - # ShareLaTeX uses nodemailer (http://www.nodemailer.com/) to send transactional emails. - # To see the range of transport and options they support, see http://www.nodemailer.com/docs/transports - # email: - # fromAddress: "" - # replyTo: "" - # lifecycle: false - # transport: "SES" - # parameters: - # AWSAccessKeyID: "" - # AWSSecretKey: "" + # Where your instance of ShareLaTeX can be found publicly. This is used + # when emails are sent out and in generated links: + siteUrl : 'http://localhost:3000' + + # If provided, a sessionSecret is used to sign cookies so that they cannot be + # spoofed. This is recommended. + security: + sessionSecret: "CRYPTO_RANDOM" # This was randomly generated for you - # Third party services - # -------------------- - # - # ShareLaTeX's regular newsletter is managed by Markdown mail. Add your - # credentials here to integrate with this. - # markdownmail: - # secret: "" - # list_id: "" - # - # Fill in your unique token from various analytics services to enable - # them. - # analytics: - # mixpanel: - # token: "" - # ga: - # token: "" - # heap: - # token: "" - # - # ShareLaTeX's help desk is provided by tenderapp.com - # tenderUrl: "" - # - - # Production Settings - # ------------------- - + # These credentials are used for authenticating api requests + # between services that may need to go over public channels + httpAuthUsers: httpAuthUsers + # Should javascript assets be served minified or not. Note that you will # need to run `grunt compile:minify` within the web-sharelatex directory # to generate these. useMinifiedJs: false # Should static assets be sent with a header to tell the browser to cache - # them. + # them. This should be false in development where changes are being made, + # but should be set to true in production. cacheStaticAssets: false # If you are running ShareLaTeX over https, set this to true to send the # cookie with a secure flag (recommended). secureCookie: false - - # Internal configs - # ---------------- - path: - # If we ever need to write something to disk (e.g. incoming requests - # that need processing but may be too big for memory, then write - # them to disk here). - dumpFolder: Path.resolve "data/dumpFolder" - # Where to write the project to disk before running LaTeX on it - compilesDir: Path.resolve(__dirname + "/../compiles") - # Where to cache downloaded URLs for the CLSI - clsiCacheDir: Path.resolve(__dirname + "/../cache") - # Automatic Snapshots - # ------------------- - automaticSnapshots: - # How long should we wait after the user last edited to - # take a snapshot? - waitTimeAfterLastEdit: 5 * minutes - # Even if edits are still taking place, this is maximum - # time to wait before taking another snapshot. - maxTimeBetweenSnapshots: 30 * minutes + # If you are running ShareLaTeX behind a proxy (like Apache, Nginx, etc) + # then set this to true to allow it to correctly detect the forwarded IP + # address and http/https protocol information. + behindProxy: false - # Smoke test - # ---------- - # Provide log in credentials and a project to be able to run - # some basic smoke tests to check the core functionality. + # Sending Email + # ------------- # - # smokeTest: - # user: "" - # password: "" - # projectId: "" + # You must configure a mail server to be able to send invite emails from + # ShareLaTeX. The config settings are passed to nodemailer. See the nodemailer + # documentation for available options: + # + # http://www.nodemailer.com/docs/transports + # + # email: + # fromAddress: "" + # replyTo: "" + # transport: "SES" + # parameters: + # AWSAccessKeyID: "" + # AWSSecretKey: "" - # Filestore health check - # ---------------------- - # Project and file details to check in filestore when calling /health_check - # health_check: - # project_id: "" - # file_id: "" + # Spell Check Languages + # --------------------- + # + # You must have the corresponding aspell dictionary installed to + # be able to use a language. Run `grunt check:aspell` to check which + # dictionaries you have installed. These should be set for the `code` for + # each language. + languages: [ + {name: "English", code: "en"} + ] + + # Service locations + # ----------------- + + # ShareLaTeX is comprised of many small services, which each expose + # an HTTP API running on a different port. Generally you + # can leave these as they are unless you have some other services + # running which conflict, or want to run the web process on port 80. + # internal: + # web: + # port: webPort = 3000 + # host: "localhost" + # documentupdater: + # port: docUpdaterPort = 3003 + # host: "localhost" + # filestore: + # port: filestorePort = 3009 + # host: "localhost" + # chat: + # port: chatPort = 3010 + # host: "localhost" + # tags: + # port: tagsPort = 3012 + # host: "localhost" + # clsi: + # port: clsiPort = 3013 + # host: "localhost" + # trackchanges: + # port: trackchangesPort = 3015 + # host: "localhost" + # docstore: + # port: docstorePort = 3016 + # host: "localhost" + # spelling: + # port: spellingPort = 3005 + # host: "localhost" + + # If you change the above config, or run some services on remote servers, + # you need to tell the other services where to find them: + apis: + web: + url: "http://localhost:3000" + user: httpAuthUser + pass: httpAuthPass + # documentupdater: + # url : "http://localhost:#{docUpdaterPort}" + # clsi: + # url: "http://localhost:#{clsiPort}" + # filestore: + # url: "http://localhost:#{filestorePort}" + # trackchanges: + # url: "http://localhost:#{trackchangesPort}" + # docstore: + # url: "http://localhost:#{docstorePort}" + # tags: + # url: "http://localhost:#{tagsPort}" + # spelling: + # url: "http://localhost:#{spellingPort}" + # chat: + # url: "http://localhost:#{chatPort}" + + +# With lots of incoming and outgoing HTTP connections to different services, +# sometimes long running, it is a good idea to increase the default number +# of sockets that Node will hold open. +http = require('http') +http.globalAgent.maxSockets = 300 +https = require('https') +https.globalAgent.maxSockets = 300 diff --git a/server-ce/package.json b/server-ce/package.json index a36305c5e1..2e922d5b14 100644 --- a/server-ce/package.json +++ b/server-ce/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "description": "An online collaborative LaTeX editor", "dependencies": { + "async": "^0.9.0", "rimraf": "~2.2.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git" }, diff --git a/server-ce/user_files/.gitignore b/server-ce/user_files/.gitignore deleted file mode 100644 index e69de29bb2..0000000000