diff --git a/server-ce/genScript.js b/server-ce/genScript.js index 230fa20bc1..dfc3ea77f2 100644 --- a/server-ce/genScript.js +++ b/server-ce/genScript.js @@ -23,6 +23,8 @@ switch (process.argv.pop()) { console.log('rm -rf node_modules/.cache') // uninstall webpack and frontend dependencies console.log('npm install --omit=dev') + // precompile pug + console.log('npm run precompile-pug') break default: console.log(`echo ${service.name} does not require a compilation`) diff --git a/services/web/.gitignore b/services/web/.gitignore index 801187d4a4..8bd23b7f0a 100644 --- a/services/web/.gitignore +++ b/services/web/.gitignore @@ -72,6 +72,9 @@ config/*.js modules/**/Makefile +# Precompiled pug files +**/app/views/**/*.js + # Sentry secrets file (injected by CI) .sentryclirc diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 3b57ccb0f7..1e85c801fb 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -20,6 +20,7 @@ RUN mkdir -p /overleaf/services/web/data/dumpFolder \ && chmod -R 0755 /overleaf/services/web/data \ && chown -R node:node /overleaf/services/web/data + # the deps image is used for caching npm ci FROM base as deps-prod @@ -37,13 +38,6 @@ ENV CYPRESS_INSTALL_BINARY=0 COPY tsconfig.backend.json /overleaf/ RUN cd /overleaf && npm install -# the web image with only production dependencies but no webpack production build, for development -FROM deps-prod as app-only - -COPY services/web /overleaf/services/web -USER node - -CMD ["node", "--expose-gc", "app.js"] # the dev is suitable for running tests FROM deps as dev @@ -60,22 +54,33 @@ USER node # the webpack image has deps+src+webpack artifacts FROM dev as webpack - USER root RUN chmod 0755 ./install_deps.sh && ./install_deps.sh -# the final production image without webpack source maps +# intermediate image for removing source maps ahead of copying into final production image FROM webpack as webpack-no-sourcemaps RUN find /overleaf/services/web/public -name '*.js.map' -delete -FROM deps-prod as app +# copy source code and precompile pug images +FROM deps-prod as pug +COPY services/web /overleaf/services/web +# Omit Server Pro/CE specific scripts from SaaS image +RUN rm /overleaf/services/web/modules/server-ce-scripts -rf +RUN OVERLEAF_CONFIG=/overleaf/services/web/config/settings.overrides.saas.js npm run precompile-pug + + +# the web image with only production dependencies but no webpack production build, for development +FROM pug as app-only +USER node +CMD ["node", "--expose-gc", "app.js"] + + +# the final production image, with webpack production build but without source maps +FROM pug as app ARG SENTRY_RELEASE ENV SENTRY_RELEASE=$SENTRY_RELEASE -COPY services/web /overleaf/services/web COPY --from=webpack-no-sourcemaps /overleaf/services/web/public /overleaf/services/web/public -RUN rm /overleaf/services/web/modules/server-ce-scripts -rf USER node - CMD ["node", "--expose-gc", "app.js"] diff --git a/services/web/Makefile b/services/web/Makefile index f7c24f4aec..22b17f8bd8 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -492,10 +492,20 @@ build_webpack_once: --file Dockerfile \ ../.. +build_pug: + docker build \ + --build-arg SENTRY_RELEASE \ + --cache-from $(IMAGE_CI)-dev \ + --tag $(IMAGE_CI)-pug \ + --target pug \ + --file Dockerfile \ + ../.. + build: docker build \ --build-arg SENTRY_RELEASE \ --cache-from $(IMAGE_CI)-webpack \ + --cache-from $(IMAGE_CI)-pug \ --cache-from $(IMAGE_REPO_FINAL) \ --tag $(IMAGE_REPO_FINAL) \ --target app \ diff --git a/services/web/app/src/infrastructure/ExpressLocals.js b/services/web/app/src/infrastructure/ExpressLocals.js index c7dcb9eebb..4c2d227c33 100644 --- a/services/web/app/src/infrastructure/ExpressLocals.js +++ b/services/web/app/src/infrastructure/ExpressLocals.js @@ -314,7 +314,7 @@ module.exports = function (webRouter, privateApiRouter, publicApiRouter) { webRouter.use(function (req, res, next) { if (Settings.reloadModuleViewsOnEachRequest) { - Modules.loadViewIncludes() + Modules.loadViewIncludes(req.app) } res.locals.moduleIncludes = Modules.moduleIncludes res.locals.moduleIncludesAvailable = Modules.moduleIncludesAvailable diff --git a/services/web/app/src/infrastructure/Modules.js b/services/web/app/src/infrastructure/Modules.js index 54fe261256..593b0bbcc9 100644 --- a/services/web/app/src/infrastructure/Modules.js +++ b/services/web/app/src/infrastructure/Modules.js @@ -1,9 +1,9 @@ const fs = require('fs') const Path = require('path') -const pug = require('pug') const async = require('async') const { promisify } = require('util') const Settings = require('@overleaf/settings') +const Views = require('./Views') const MODULE_BASE_PATH = Path.join(__dirname, '/../../../modules') @@ -35,6 +35,11 @@ function loadModules() { ) loadedModule.name = moduleName _modules.push(loadedModule) + if (loadedModule.viewIncludes) { + throw new Error( + `${moduleName}: module.viewIncludes moved into Settings.viewIncludes` + ) + } } _modulesLoaded = true attachHooks() @@ -64,28 +69,7 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) { } function loadViewIncludes(app) { - _viewIncludes = {} - for (const module of modules()) { - const object = module.viewIncludes || {} - for (const view in object) { - const partial = object[view] - if (!_viewIncludes[view]) { - _viewIncludes[view] = [] - } - const filePath = Path.join( - MODULE_BASE_PATH, - module.name, - 'app/views', - partial + '.pug' - ) - _viewIncludes[view].push( - pug.compileFile(filePath, { - doctype: 'html', - compileDebug: Settings.debugPugTemplates, - }) - ) - } - } + _viewIncludes = Views.compileViewIncludes(app) } function registerMiddleware(appOrRouter, middlewareName, options) { diff --git a/services/web/app/src/infrastructure/Server.js b/services/web/app/src/infrastructure/Server.js index 0830a6f89d..43ae580ec4 100644 --- a/services/web/app/src/infrastructure/Server.js +++ b/services/web/app/src/infrastructure/Server.js @@ -129,7 +129,18 @@ webRouter.use( ) app.set('views', Path.join(__dirname, '/../../views')) app.set('view engine', 'pug') -Modules.loadViewIncludes(app) + +if (Settings.enabledServices.includes('web')) { + if (app.get('env') !== 'development') { + logger.debug('enabling view cache for production or acceptance tests') + app.enable('view cache') + } + if (Settings.precompilePugTemplatesAtBootTime) { + logger.debug('precompiling views for web in production environment') + Views.precompileViews(app) + } + Modules.loadViewIncludes(app) +} app.use(metrics.http.monitor(logger)) @@ -334,16 +345,6 @@ if (Settings.enabledServices.includes('api')) { if (Settings.enabledServices.includes('web')) { logger.debug('providing web router') - - if (Settings.precompilePugTemplatesAtBootTime) { - logger.debug('precompiling views for web in production environment') - Views.precompileViews(app) - } - if (app.get('env') === 'test') { - logger.debug('enabling view cache for acceptance tests') - app.enable('view cache') - } - app.use(publicApiRouter) // public API goes with web router for public access app.use(Validation.errorMiddleware) app.use(ErrorController.handleApiError) diff --git a/services/web/app/src/infrastructure/Views.js b/services/web/app/src/infrastructure/Views.js index df3725e669..8574e1736a 100644 --- a/services/web/app/src/infrastructure/Views.js +++ b/services/web/app/src/infrastructure/Views.js @@ -2,52 +2,129 @@ const logger = require('@overleaf/logger') const pug = require('pug') const globby = require('globby') const Settings = require('@overleaf/settings') -const path = require('path') +const fs = require('fs') +const Path = require('path') // Generate list of view names from app/views - -const viewList = globby - .sync('app/views/**/*.pug', { - onlyFiles: true, - concurrency: 1, - ignore: '**/_*.pug', - }) - .concat( - globby.sync('modules/*/app/views/**/*.pug', { +function buildViewList() { + return globby + .sync('app/views/**/*.pug', { onlyFiles: true, concurrency: 1, - ignore: '**/_*.pug', + ignore: [ + // Ignore includes + '**/_*.pug', + '**/_*/**', + // Ignore shared layout files + 'app/views/layout*', + 'app/views/layout/*', + ], }) + .concat( + globby.sync('modules/*/app/views/**/*.pug', { + onlyFiles: true, + concurrency: 1, + // Ignore includes + ignore: ['**/_*.pug', '**/_*/**'], + }) + ) + .concat(Object.values(Settings.viewIncludes).flat()) + .map(x => Path.resolve(x)) +} + +const PUG_COMPILE_ARGUMENTS = { + doctype: 'html', + cache: true, + compileDebug: Settings.debugPugTemplates, + inlineRuntimeFunctions: false, + module: true, +} + +function precompileViewsAndCacheToDisk() { + const startTime = Date.now() + let success = 0 + let precompiled = 0 + for (const filename of buildViewList()) { + const precompiledFilename = filename.replace(/\.pug$/, '.js') + try { + const src = pug.compileFileClient(filename, PUG_COMPILE_ARGUMENTS) + try { + if (fs.readFileSync(precompiledFilename, 'utf-8') === src) { + precompiled++ + continue + } + } catch {} + fs.writeFileSync(precompiledFilename, src, { + encoding: 'utf-8', + mode: 0o644, + }) + success++ + } catch (err) { + logger.err({ err, filename }, 'failed to precompile pug template') + throw err + } + } + logger.info( + { timeTaken: Date.now() - startTime, success, precompiled }, + 'compiled pug templates' ) - .map(x => { - return x.replace(/\.pug$/, '') // strip trailing .pug extension - }) - .filter(x => { - return !/^_/.test(x) - }) +} module.exports = { + compileViewIncludes(app) { + const viewIncludes = {} + for (const [view, paths] of Object.entries(Settings.viewIncludes)) { + viewIncludes[view] = [] + for (const filePath of paths) { + viewIncludes[view].push( + pug.compileFile(filePath, { + ...PUG_COMPILE_ARGUMENTS, + cache: app.enabled('view cache'), + }) + ) + } + } + return viewIncludes + }, + precompileViews(app) { const startTime = Date.now() let success = 0 + let precompiled = 0 let failures = 0 - viewList.forEach(view => { - const filename = path.resolve(view + '.pug') // express views are cached using the absolute path + for (const filename of buildViewList()) { + const precompiledFilename = filename.replace(/\.pug$/, '.js') + if (fs.existsSync(precompiledFilename)) { + logger.debug({ filename }, 'loading precompiled pug template') + try { + pug.cache[filename] = require(precompiledFilename) + precompiled++ + continue + } catch (err) { + logger.error( + { filename, err }, + 'error loading precompiled pug template' + ) + failures++ + } + } try { - pug.compileFile(filename, { - cache: true, - compileDebug: Settings.debugPugTemplates, - }) - logger.debug({ filename }, 'compiled') + logger.warn({ filename }, 'compiling pug template at boot time') + pug.compileFile(filename, PUG_COMPILE_ARGUMENTS) success++ } catch (err) { - logger.error({ filename, err: err.message }, 'error compiling') + logger.error({ filename, err }, 'error compiling pug template') failures++ } - }) + } logger.debug( - { timeTaken: Date.now() - startTime, failures, success }, - 'compiled templates' + { timeTaken: Date.now() - startTime, failures, success, precompiled }, + 'compiled pug templates' ) }, } + +if (require.main === module) { + precompileViewsAndCacheToDisk() + process.exit(0) +} diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index f685ea2328..e95e974393 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -900,6 +900,7 @@ module.exports = { 'server-ce-scripts', 'user-activate', ], + viewIncludes: {}, csp: { enabled: process.env.CSP_ENABLED === 'true', diff --git a/services/web/package.json b/services/web/package.json index 02391045ee..dba7f76600 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -43,6 +43,7 @@ "routes": "bin/routes", "storybook": "storybook dev -p 6006", "build-storybook": "storybook build", + "precompile-pug": "node app/src/infrastructure/Views", "local:nodemon": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; echo $OVERLEAF_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon", "local:webpack": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; PORT=13808 OVERLEAF_CONFIG=$(pwd)/config/settings.webpack.js npm run webpack", "local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",