mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #19220 from overleaf/jpa-precompile-pug
[web] precompile pug templates in CI GitOrigin-RevId: 6ec2b85a357fa3d5c35d8e7eb1a2e81ac5f3b447
This commit is contained in:
parent
1359d7d326
commit
88457a6655
10 changed files with 160 additions and 76 deletions
|
@ -23,6 +23,8 @@ switch (process.argv.pop()) {
|
|||
console.log('rm -rf node_modules/.cache')
|
||||
// uninstall webpack and frontend dependencies
|
||||
console.log('npm install --omit=dev')
|
||||
// precompile pug
|
||||
console.log('npm run precompile-pug')
|
||||
break
|
||||
default:
|
||||
console.log(`echo ${service.name} does not require a compilation`)
|
||||
|
|
3
services/web/.gitignore
vendored
3
services/web/.gitignore
vendored
|
@ -72,6 +72,9 @@ config/*.js
|
|||
|
||||
modules/**/Makefile
|
||||
|
||||
# Precompiled pug files
|
||||
**/app/views/**/*.js
|
||||
|
||||
# Sentry secrets file (injected by CI)
|
||||
.sentryclirc
|
||||
|
||||
|
|
|
@ -20,6 +20,7 @@ RUN mkdir -p /overleaf/services/web/data/dumpFolder \
|
|||
&& chmod -R 0755 /overleaf/services/web/data \
|
||||
&& chown -R node:node /overleaf/services/web/data
|
||||
|
||||
|
||||
# the deps image is used for caching npm ci
|
||||
FROM base as deps-prod
|
||||
|
||||
|
@ -37,13 +38,6 @@ ENV CYPRESS_INSTALL_BINARY=0
|
|||
COPY tsconfig.backend.json /overleaf/
|
||||
RUN cd /overleaf && npm install
|
||||
|
||||
# the web image with only production dependencies but no webpack production build, for development
|
||||
FROM deps-prod as app-only
|
||||
|
||||
COPY services/web /overleaf/services/web
|
||||
USER node
|
||||
|
||||
CMD ["node", "--expose-gc", "app.js"]
|
||||
|
||||
# the dev is suitable for running tests
|
||||
FROM deps as dev
|
||||
|
@ -60,22 +54,33 @@ USER node
|
|||
|
||||
# the webpack image has deps+src+webpack artifacts
|
||||
FROM dev as webpack
|
||||
|
||||
USER root
|
||||
RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
|
||||
|
||||
|
||||
# the final production image without webpack source maps
|
||||
# intermediate image for removing source maps ahead of copying into final production image
|
||||
FROM webpack as webpack-no-sourcemaps
|
||||
RUN find /overleaf/services/web/public -name '*.js.map' -delete
|
||||
|
||||
FROM deps-prod as app
|
||||
|
||||
# copy source code and precompile pug images
|
||||
FROM deps-prod as pug
|
||||
COPY services/web /overleaf/services/web
|
||||
# Omit Server Pro/CE specific scripts from SaaS image
|
||||
RUN rm /overleaf/services/web/modules/server-ce-scripts -rf
|
||||
RUN OVERLEAF_CONFIG=/overleaf/services/web/config/settings.overrides.saas.js npm run precompile-pug
|
||||
|
||||
|
||||
# the web image with only production dependencies but no webpack production build, for development
|
||||
FROM pug as app-only
|
||||
USER node
|
||||
CMD ["node", "--expose-gc", "app.js"]
|
||||
|
||||
|
||||
# the final production image, with webpack production build but without source maps
|
||||
FROM pug as app
|
||||
ARG SENTRY_RELEASE
|
||||
ENV SENTRY_RELEASE=$SENTRY_RELEASE
|
||||
COPY services/web /overleaf/services/web
|
||||
COPY --from=webpack-no-sourcemaps /overleaf/services/web/public /overleaf/services/web/public
|
||||
RUN rm /overleaf/services/web/modules/server-ce-scripts -rf
|
||||
USER node
|
||||
|
||||
CMD ["node", "--expose-gc", "app.js"]
|
||||
|
|
|
@ -492,10 +492,20 @@ build_webpack_once:
|
|||
--file Dockerfile \
|
||||
../..
|
||||
|
||||
build_pug:
|
||||
docker build \
|
||||
--build-arg SENTRY_RELEASE \
|
||||
--cache-from $(IMAGE_CI)-dev \
|
||||
--tag $(IMAGE_CI)-pug \
|
||||
--target pug \
|
||||
--file Dockerfile \
|
||||
../..
|
||||
|
||||
build:
|
||||
docker build \
|
||||
--build-arg SENTRY_RELEASE \
|
||||
--cache-from $(IMAGE_CI)-webpack \
|
||||
--cache-from $(IMAGE_CI)-pug \
|
||||
--cache-from $(IMAGE_REPO_FINAL) \
|
||||
--tag $(IMAGE_REPO_FINAL) \
|
||||
--target app \
|
||||
|
|
|
@ -314,7 +314,7 @@ module.exports = function (webRouter, privateApiRouter, publicApiRouter) {
|
|||
|
||||
webRouter.use(function (req, res, next) {
|
||||
if (Settings.reloadModuleViewsOnEachRequest) {
|
||||
Modules.loadViewIncludes()
|
||||
Modules.loadViewIncludes(req.app)
|
||||
}
|
||||
res.locals.moduleIncludes = Modules.moduleIncludes
|
||||
res.locals.moduleIncludesAvailable = Modules.moduleIncludesAvailable
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
const pug = require('pug')
|
||||
const async = require('async')
|
||||
const { promisify } = require('util')
|
||||
const Settings = require('@overleaf/settings')
|
||||
const Views = require('./Views')
|
||||
|
||||
const MODULE_BASE_PATH = Path.join(__dirname, '/../../../modules')
|
||||
|
||||
|
@ -35,6 +35,11 @@ function loadModules() {
|
|||
)
|
||||
loadedModule.name = moduleName
|
||||
_modules.push(loadedModule)
|
||||
if (loadedModule.viewIncludes) {
|
||||
throw new Error(
|
||||
`${moduleName}: module.viewIncludes moved into Settings.viewIncludes`
|
||||
)
|
||||
}
|
||||
}
|
||||
_modulesLoaded = true
|
||||
attachHooks()
|
||||
|
@ -64,28 +69,7 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) {
|
|||
}
|
||||
|
||||
function loadViewIncludes(app) {
|
||||
_viewIncludes = {}
|
||||
for (const module of modules()) {
|
||||
const object = module.viewIncludes || {}
|
||||
for (const view in object) {
|
||||
const partial = object[view]
|
||||
if (!_viewIncludes[view]) {
|
||||
_viewIncludes[view] = []
|
||||
}
|
||||
const filePath = Path.join(
|
||||
MODULE_BASE_PATH,
|
||||
module.name,
|
||||
'app/views',
|
||||
partial + '.pug'
|
||||
)
|
||||
_viewIncludes[view].push(
|
||||
pug.compileFile(filePath, {
|
||||
doctype: 'html',
|
||||
compileDebug: Settings.debugPugTemplates,
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
_viewIncludes = Views.compileViewIncludes(app)
|
||||
}
|
||||
|
||||
function registerMiddleware(appOrRouter, middlewareName, options) {
|
||||
|
|
|
@ -129,7 +129,18 @@ webRouter.use(
|
|||
)
|
||||
app.set('views', Path.join(__dirname, '/../../views'))
|
||||
app.set('view engine', 'pug')
|
||||
Modules.loadViewIncludes(app)
|
||||
|
||||
if (Settings.enabledServices.includes('web')) {
|
||||
if (app.get('env') !== 'development') {
|
||||
logger.debug('enabling view cache for production or acceptance tests')
|
||||
app.enable('view cache')
|
||||
}
|
||||
if (Settings.precompilePugTemplatesAtBootTime) {
|
||||
logger.debug('precompiling views for web in production environment')
|
||||
Views.precompileViews(app)
|
||||
}
|
||||
Modules.loadViewIncludes(app)
|
||||
}
|
||||
|
||||
app.use(metrics.http.monitor(logger))
|
||||
|
||||
|
@ -334,16 +345,6 @@ if (Settings.enabledServices.includes('api')) {
|
|||
|
||||
if (Settings.enabledServices.includes('web')) {
|
||||
logger.debug('providing web router')
|
||||
|
||||
if (Settings.precompilePugTemplatesAtBootTime) {
|
||||
logger.debug('precompiling views for web in production environment')
|
||||
Views.precompileViews(app)
|
||||
}
|
||||
if (app.get('env') === 'test') {
|
||||
logger.debug('enabling view cache for acceptance tests')
|
||||
app.enable('view cache')
|
||||
}
|
||||
|
||||
app.use(publicApiRouter) // public API goes with web router for public access
|
||||
app.use(Validation.errorMiddleware)
|
||||
app.use(ErrorController.handleApiError)
|
||||
|
|
|
@ -2,52 +2,129 @@ const logger = require('@overleaf/logger')
|
|||
const pug = require('pug')
|
||||
const globby = require('globby')
|
||||
const Settings = require('@overleaf/settings')
|
||||
const path = require('path')
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
|
||||
// Generate list of view names from app/views
|
||||
|
||||
const viewList = globby
|
||||
.sync('app/views/**/*.pug', {
|
||||
onlyFiles: true,
|
||||
concurrency: 1,
|
||||
ignore: '**/_*.pug',
|
||||
})
|
||||
.concat(
|
||||
globby.sync('modules/*/app/views/**/*.pug', {
|
||||
function buildViewList() {
|
||||
return globby
|
||||
.sync('app/views/**/*.pug', {
|
||||
onlyFiles: true,
|
||||
concurrency: 1,
|
||||
ignore: '**/_*.pug',
|
||||
ignore: [
|
||||
// Ignore includes
|
||||
'**/_*.pug',
|
||||
'**/_*/**',
|
||||
// Ignore shared layout files
|
||||
'app/views/layout*',
|
||||
'app/views/layout/*',
|
||||
],
|
||||
})
|
||||
.concat(
|
||||
globby.sync('modules/*/app/views/**/*.pug', {
|
||||
onlyFiles: true,
|
||||
concurrency: 1,
|
||||
// Ignore includes
|
||||
ignore: ['**/_*.pug', '**/_*/**'],
|
||||
})
|
||||
)
|
||||
.concat(Object.values(Settings.viewIncludes).flat())
|
||||
.map(x => Path.resolve(x))
|
||||
}
|
||||
|
||||
const PUG_COMPILE_ARGUMENTS = {
|
||||
doctype: 'html',
|
||||
cache: true,
|
||||
compileDebug: Settings.debugPugTemplates,
|
||||
inlineRuntimeFunctions: false,
|
||||
module: true,
|
||||
}
|
||||
|
||||
function precompileViewsAndCacheToDisk() {
|
||||
const startTime = Date.now()
|
||||
let success = 0
|
||||
let precompiled = 0
|
||||
for (const filename of buildViewList()) {
|
||||
const precompiledFilename = filename.replace(/\.pug$/, '.js')
|
||||
try {
|
||||
const src = pug.compileFileClient(filename, PUG_COMPILE_ARGUMENTS)
|
||||
try {
|
||||
if (fs.readFileSync(precompiledFilename, 'utf-8') === src) {
|
||||
precompiled++
|
||||
continue
|
||||
}
|
||||
} catch {}
|
||||
fs.writeFileSync(precompiledFilename, src, {
|
||||
encoding: 'utf-8',
|
||||
mode: 0o644,
|
||||
})
|
||||
success++
|
||||
} catch (err) {
|
||||
logger.err({ err, filename }, 'failed to precompile pug template')
|
||||
throw err
|
||||
}
|
||||
}
|
||||
logger.info(
|
||||
{ timeTaken: Date.now() - startTime, success, precompiled },
|
||||
'compiled pug templates'
|
||||
)
|
||||
.map(x => {
|
||||
return x.replace(/\.pug$/, '') // strip trailing .pug extension
|
||||
})
|
||||
.filter(x => {
|
||||
return !/^_/.test(x)
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
compileViewIncludes(app) {
|
||||
const viewIncludes = {}
|
||||
for (const [view, paths] of Object.entries(Settings.viewIncludes)) {
|
||||
viewIncludes[view] = []
|
||||
for (const filePath of paths) {
|
||||
viewIncludes[view].push(
|
||||
pug.compileFile(filePath, {
|
||||
...PUG_COMPILE_ARGUMENTS,
|
||||
cache: app.enabled('view cache'),
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
return viewIncludes
|
||||
},
|
||||
|
||||
precompileViews(app) {
|
||||
const startTime = Date.now()
|
||||
let success = 0
|
||||
let precompiled = 0
|
||||
let failures = 0
|
||||
viewList.forEach(view => {
|
||||
const filename = path.resolve(view + '.pug') // express views are cached using the absolute path
|
||||
for (const filename of buildViewList()) {
|
||||
const precompiledFilename = filename.replace(/\.pug$/, '.js')
|
||||
if (fs.existsSync(precompiledFilename)) {
|
||||
logger.debug({ filename }, 'loading precompiled pug template')
|
||||
try {
|
||||
pug.cache[filename] = require(precompiledFilename)
|
||||
precompiled++
|
||||
continue
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ filename, err },
|
||||
'error loading precompiled pug template'
|
||||
)
|
||||
failures++
|
||||
}
|
||||
}
|
||||
try {
|
||||
pug.compileFile(filename, {
|
||||
cache: true,
|
||||
compileDebug: Settings.debugPugTemplates,
|
||||
})
|
||||
logger.debug({ filename }, 'compiled')
|
||||
logger.warn({ filename }, 'compiling pug template at boot time')
|
||||
pug.compileFile(filename, PUG_COMPILE_ARGUMENTS)
|
||||
success++
|
||||
} catch (err) {
|
||||
logger.error({ filename, err: err.message }, 'error compiling')
|
||||
logger.error({ filename, err }, 'error compiling pug template')
|
||||
failures++
|
||||
}
|
||||
})
|
||||
}
|
||||
logger.debug(
|
||||
{ timeTaken: Date.now() - startTime, failures, success },
|
||||
'compiled templates'
|
||||
{ timeTaken: Date.now() - startTime, failures, success, precompiled },
|
||||
'compiled pug templates'
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
precompileViewsAndCacheToDisk()
|
||||
process.exit(0)
|
||||
}
|
||||
|
|
|
@ -900,6 +900,7 @@ module.exports = {
|
|||
'server-ce-scripts',
|
||||
'user-activate',
|
||||
],
|
||||
viewIncludes: {},
|
||||
|
||||
csp: {
|
||||
enabled: process.env.CSP_ENABLED === 'true',
|
||||
|
|
|
@ -43,6 +43,7 @@
|
|||
"routes": "bin/routes",
|
||||
"storybook": "storybook dev -p 6006",
|
||||
"build-storybook": "storybook build",
|
||||
"precompile-pug": "node app/src/infrastructure/Views",
|
||||
"local:nodemon": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; echo $OVERLEAF_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon",
|
||||
"local:webpack": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; PORT=13808 OVERLEAF_CONFIG=$(pwd)/config/settings.webpack.js npm run webpack",
|
||||
"local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",
|
||||
|
|
Loading…
Reference in a new issue