mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-14 20:40:17 -05:00
Merge pull request #19220 from overleaf/jpa-precompile-pug
[web] precompile pug templates in CI GitOrigin-RevId: 6ec2b85a357fa3d5c35d8e7eb1a2e81ac5f3b447
This commit is contained in:
parent
1359d7d326
commit
88457a6655
10 changed files with 160 additions and 76 deletions
|
@ -23,6 +23,8 @@ switch (process.argv.pop()) {
|
||||||
console.log('rm -rf node_modules/.cache')
|
console.log('rm -rf node_modules/.cache')
|
||||||
// uninstall webpack and frontend dependencies
|
// uninstall webpack and frontend dependencies
|
||||||
console.log('npm install --omit=dev')
|
console.log('npm install --omit=dev')
|
||||||
|
// precompile pug
|
||||||
|
console.log('npm run precompile-pug')
|
||||||
break
|
break
|
||||||
default:
|
default:
|
||||||
console.log(`echo ${service.name} does not require a compilation`)
|
console.log(`echo ${service.name} does not require a compilation`)
|
||||||
|
|
3
services/web/.gitignore
vendored
3
services/web/.gitignore
vendored
|
@ -72,6 +72,9 @@ config/*.js
|
||||||
|
|
||||||
modules/**/Makefile
|
modules/**/Makefile
|
||||||
|
|
||||||
|
# Precompiled pug files
|
||||||
|
**/app/views/**/*.js
|
||||||
|
|
||||||
# Sentry secrets file (injected by CI)
|
# Sentry secrets file (injected by CI)
|
||||||
.sentryclirc
|
.sentryclirc
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@ RUN mkdir -p /overleaf/services/web/data/dumpFolder \
|
||||||
&& chmod -R 0755 /overleaf/services/web/data \
|
&& chmod -R 0755 /overleaf/services/web/data \
|
||||||
&& chown -R node:node /overleaf/services/web/data
|
&& chown -R node:node /overleaf/services/web/data
|
||||||
|
|
||||||
|
|
||||||
# the deps image is used for caching npm ci
|
# the deps image is used for caching npm ci
|
||||||
FROM base as deps-prod
|
FROM base as deps-prod
|
||||||
|
|
||||||
|
@ -37,13 +38,6 @@ ENV CYPRESS_INSTALL_BINARY=0
|
||||||
COPY tsconfig.backend.json /overleaf/
|
COPY tsconfig.backend.json /overleaf/
|
||||||
RUN cd /overleaf && npm install
|
RUN cd /overleaf && npm install
|
||||||
|
|
||||||
# the web image with only production dependencies but no webpack production build, for development
|
|
||||||
FROM deps-prod as app-only
|
|
||||||
|
|
||||||
COPY services/web /overleaf/services/web
|
|
||||||
USER node
|
|
||||||
|
|
||||||
CMD ["node", "--expose-gc", "app.js"]
|
|
||||||
|
|
||||||
# the dev is suitable for running tests
|
# the dev is suitable for running tests
|
||||||
FROM deps as dev
|
FROM deps as dev
|
||||||
|
@ -60,22 +54,33 @@ USER node
|
||||||
|
|
||||||
# the webpack image has deps+src+webpack artifacts
|
# the webpack image has deps+src+webpack artifacts
|
||||||
FROM dev as webpack
|
FROM dev as webpack
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
|
RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
|
||||||
|
|
||||||
|
|
||||||
# the final production image without webpack source maps
|
# intermediate image for removing source maps ahead of copying into final production image
|
||||||
FROM webpack as webpack-no-sourcemaps
|
FROM webpack as webpack-no-sourcemaps
|
||||||
RUN find /overleaf/services/web/public -name '*.js.map' -delete
|
RUN find /overleaf/services/web/public -name '*.js.map' -delete
|
||||||
|
|
||||||
FROM deps-prod as app
|
|
||||||
|
|
||||||
|
# copy source code and precompile pug images
|
||||||
|
FROM deps-prod as pug
|
||||||
|
COPY services/web /overleaf/services/web
|
||||||
|
# Omit Server Pro/CE specific scripts from SaaS image
|
||||||
|
RUN rm /overleaf/services/web/modules/server-ce-scripts -rf
|
||||||
|
RUN OVERLEAF_CONFIG=/overleaf/services/web/config/settings.overrides.saas.js npm run precompile-pug
|
||||||
|
|
||||||
|
|
||||||
|
# the web image with only production dependencies but no webpack production build, for development
|
||||||
|
FROM pug as app-only
|
||||||
|
USER node
|
||||||
|
CMD ["node", "--expose-gc", "app.js"]
|
||||||
|
|
||||||
|
|
||||||
|
# the final production image, with webpack production build but without source maps
|
||||||
|
FROM pug as app
|
||||||
ARG SENTRY_RELEASE
|
ARG SENTRY_RELEASE
|
||||||
ENV SENTRY_RELEASE=$SENTRY_RELEASE
|
ENV SENTRY_RELEASE=$SENTRY_RELEASE
|
||||||
COPY services/web /overleaf/services/web
|
|
||||||
COPY --from=webpack-no-sourcemaps /overleaf/services/web/public /overleaf/services/web/public
|
COPY --from=webpack-no-sourcemaps /overleaf/services/web/public /overleaf/services/web/public
|
||||||
RUN rm /overleaf/services/web/modules/server-ce-scripts -rf
|
|
||||||
USER node
|
USER node
|
||||||
|
|
||||||
CMD ["node", "--expose-gc", "app.js"]
|
CMD ["node", "--expose-gc", "app.js"]
|
||||||
|
|
|
@ -492,10 +492,20 @@ build_webpack_once:
|
||||||
--file Dockerfile \
|
--file Dockerfile \
|
||||||
../..
|
../..
|
||||||
|
|
||||||
|
build_pug:
|
||||||
|
docker build \
|
||||||
|
--build-arg SENTRY_RELEASE \
|
||||||
|
--cache-from $(IMAGE_CI)-dev \
|
||||||
|
--tag $(IMAGE_CI)-pug \
|
||||||
|
--target pug \
|
||||||
|
--file Dockerfile \
|
||||||
|
../..
|
||||||
|
|
||||||
build:
|
build:
|
||||||
docker build \
|
docker build \
|
||||||
--build-arg SENTRY_RELEASE \
|
--build-arg SENTRY_RELEASE \
|
||||||
--cache-from $(IMAGE_CI)-webpack \
|
--cache-from $(IMAGE_CI)-webpack \
|
||||||
|
--cache-from $(IMAGE_CI)-pug \
|
||||||
--cache-from $(IMAGE_REPO_FINAL) \
|
--cache-from $(IMAGE_REPO_FINAL) \
|
||||||
--tag $(IMAGE_REPO_FINAL) \
|
--tag $(IMAGE_REPO_FINAL) \
|
||||||
--target app \
|
--target app \
|
||||||
|
|
|
@ -314,7 +314,7 @@ module.exports = function (webRouter, privateApiRouter, publicApiRouter) {
|
||||||
|
|
||||||
webRouter.use(function (req, res, next) {
|
webRouter.use(function (req, res, next) {
|
||||||
if (Settings.reloadModuleViewsOnEachRequest) {
|
if (Settings.reloadModuleViewsOnEachRequest) {
|
||||||
Modules.loadViewIncludes()
|
Modules.loadViewIncludes(req.app)
|
||||||
}
|
}
|
||||||
res.locals.moduleIncludes = Modules.moduleIncludes
|
res.locals.moduleIncludes = Modules.moduleIncludes
|
||||||
res.locals.moduleIncludesAvailable = Modules.moduleIncludesAvailable
|
res.locals.moduleIncludesAvailable = Modules.moduleIncludesAvailable
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
const fs = require('fs')
|
const fs = require('fs')
|
||||||
const Path = require('path')
|
const Path = require('path')
|
||||||
const pug = require('pug')
|
|
||||||
const async = require('async')
|
const async = require('async')
|
||||||
const { promisify } = require('util')
|
const { promisify } = require('util')
|
||||||
const Settings = require('@overleaf/settings')
|
const Settings = require('@overleaf/settings')
|
||||||
|
const Views = require('./Views')
|
||||||
|
|
||||||
const MODULE_BASE_PATH = Path.join(__dirname, '/../../../modules')
|
const MODULE_BASE_PATH = Path.join(__dirname, '/../../../modules')
|
||||||
|
|
||||||
|
@ -35,6 +35,11 @@ function loadModules() {
|
||||||
)
|
)
|
||||||
loadedModule.name = moduleName
|
loadedModule.name = moduleName
|
||||||
_modules.push(loadedModule)
|
_modules.push(loadedModule)
|
||||||
|
if (loadedModule.viewIncludes) {
|
||||||
|
throw new Error(
|
||||||
|
`${moduleName}: module.viewIncludes moved into Settings.viewIncludes`
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
_modulesLoaded = true
|
_modulesLoaded = true
|
||||||
attachHooks()
|
attachHooks()
|
||||||
|
@ -64,28 +69,7 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadViewIncludes(app) {
|
function loadViewIncludes(app) {
|
||||||
_viewIncludes = {}
|
_viewIncludes = Views.compileViewIncludes(app)
|
||||||
for (const module of modules()) {
|
|
||||||
const object = module.viewIncludes || {}
|
|
||||||
for (const view in object) {
|
|
||||||
const partial = object[view]
|
|
||||||
if (!_viewIncludes[view]) {
|
|
||||||
_viewIncludes[view] = []
|
|
||||||
}
|
|
||||||
const filePath = Path.join(
|
|
||||||
MODULE_BASE_PATH,
|
|
||||||
module.name,
|
|
||||||
'app/views',
|
|
||||||
partial + '.pug'
|
|
||||||
)
|
|
||||||
_viewIncludes[view].push(
|
|
||||||
pug.compileFile(filePath, {
|
|
||||||
doctype: 'html',
|
|
||||||
compileDebug: Settings.debugPugTemplates,
|
|
||||||
})
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function registerMiddleware(appOrRouter, middlewareName, options) {
|
function registerMiddleware(appOrRouter, middlewareName, options) {
|
||||||
|
|
|
@ -129,7 +129,18 @@ webRouter.use(
|
||||||
)
|
)
|
||||||
app.set('views', Path.join(__dirname, '/../../views'))
|
app.set('views', Path.join(__dirname, '/../../views'))
|
||||||
app.set('view engine', 'pug')
|
app.set('view engine', 'pug')
|
||||||
Modules.loadViewIncludes(app)
|
|
||||||
|
if (Settings.enabledServices.includes('web')) {
|
||||||
|
if (app.get('env') !== 'development') {
|
||||||
|
logger.debug('enabling view cache for production or acceptance tests')
|
||||||
|
app.enable('view cache')
|
||||||
|
}
|
||||||
|
if (Settings.precompilePugTemplatesAtBootTime) {
|
||||||
|
logger.debug('precompiling views for web in production environment')
|
||||||
|
Views.precompileViews(app)
|
||||||
|
}
|
||||||
|
Modules.loadViewIncludes(app)
|
||||||
|
}
|
||||||
|
|
||||||
app.use(metrics.http.monitor(logger))
|
app.use(metrics.http.monitor(logger))
|
||||||
|
|
||||||
|
@ -334,16 +345,6 @@ if (Settings.enabledServices.includes('api')) {
|
||||||
|
|
||||||
if (Settings.enabledServices.includes('web')) {
|
if (Settings.enabledServices.includes('web')) {
|
||||||
logger.debug('providing web router')
|
logger.debug('providing web router')
|
||||||
|
|
||||||
if (Settings.precompilePugTemplatesAtBootTime) {
|
|
||||||
logger.debug('precompiling views for web in production environment')
|
|
||||||
Views.precompileViews(app)
|
|
||||||
}
|
|
||||||
if (app.get('env') === 'test') {
|
|
||||||
logger.debug('enabling view cache for acceptance tests')
|
|
||||||
app.enable('view cache')
|
|
||||||
}
|
|
||||||
|
|
||||||
app.use(publicApiRouter) // public API goes with web router for public access
|
app.use(publicApiRouter) // public API goes with web router for public access
|
||||||
app.use(Validation.errorMiddleware)
|
app.use(Validation.errorMiddleware)
|
||||||
app.use(ErrorController.handleApiError)
|
app.use(ErrorController.handleApiError)
|
||||||
|
|
|
@ -2,52 +2,129 @@ const logger = require('@overleaf/logger')
|
||||||
const pug = require('pug')
|
const pug = require('pug')
|
||||||
const globby = require('globby')
|
const globby = require('globby')
|
||||||
const Settings = require('@overleaf/settings')
|
const Settings = require('@overleaf/settings')
|
||||||
const path = require('path')
|
const fs = require('fs')
|
||||||
|
const Path = require('path')
|
||||||
|
|
||||||
// Generate list of view names from app/views
|
// Generate list of view names from app/views
|
||||||
|
function buildViewList() {
|
||||||
const viewList = globby
|
return globby
|
||||||
.sync('app/views/**/*.pug', {
|
.sync('app/views/**/*.pug', {
|
||||||
onlyFiles: true,
|
|
||||||
concurrency: 1,
|
|
||||||
ignore: '**/_*.pug',
|
|
||||||
})
|
|
||||||
.concat(
|
|
||||||
globby.sync('modules/*/app/views/**/*.pug', {
|
|
||||||
onlyFiles: true,
|
onlyFiles: true,
|
||||||
concurrency: 1,
|
concurrency: 1,
|
||||||
ignore: '**/_*.pug',
|
ignore: [
|
||||||
|
// Ignore includes
|
||||||
|
'**/_*.pug',
|
||||||
|
'**/_*/**',
|
||||||
|
// Ignore shared layout files
|
||||||
|
'app/views/layout*',
|
||||||
|
'app/views/layout/*',
|
||||||
|
],
|
||||||
})
|
})
|
||||||
|
.concat(
|
||||||
|
globby.sync('modules/*/app/views/**/*.pug', {
|
||||||
|
onlyFiles: true,
|
||||||
|
concurrency: 1,
|
||||||
|
// Ignore includes
|
||||||
|
ignore: ['**/_*.pug', '**/_*/**'],
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.concat(Object.values(Settings.viewIncludes).flat())
|
||||||
|
.map(x => Path.resolve(x))
|
||||||
|
}
|
||||||
|
|
||||||
|
const PUG_COMPILE_ARGUMENTS = {
|
||||||
|
doctype: 'html',
|
||||||
|
cache: true,
|
||||||
|
compileDebug: Settings.debugPugTemplates,
|
||||||
|
inlineRuntimeFunctions: false,
|
||||||
|
module: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
function precompileViewsAndCacheToDisk() {
|
||||||
|
const startTime = Date.now()
|
||||||
|
let success = 0
|
||||||
|
let precompiled = 0
|
||||||
|
for (const filename of buildViewList()) {
|
||||||
|
const precompiledFilename = filename.replace(/\.pug$/, '.js')
|
||||||
|
try {
|
||||||
|
const src = pug.compileFileClient(filename, PUG_COMPILE_ARGUMENTS)
|
||||||
|
try {
|
||||||
|
if (fs.readFileSync(precompiledFilename, 'utf-8') === src) {
|
||||||
|
precompiled++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
fs.writeFileSync(precompiledFilename, src, {
|
||||||
|
encoding: 'utf-8',
|
||||||
|
mode: 0o644,
|
||||||
|
})
|
||||||
|
success++
|
||||||
|
} catch (err) {
|
||||||
|
logger.err({ err, filename }, 'failed to precompile pug template')
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.info(
|
||||||
|
{ timeTaken: Date.now() - startTime, success, precompiled },
|
||||||
|
'compiled pug templates'
|
||||||
)
|
)
|
||||||
.map(x => {
|
}
|
||||||
return x.replace(/\.pug$/, '') // strip trailing .pug extension
|
|
||||||
})
|
|
||||||
.filter(x => {
|
|
||||||
return !/^_/.test(x)
|
|
||||||
})
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
compileViewIncludes(app) {
|
||||||
|
const viewIncludes = {}
|
||||||
|
for (const [view, paths] of Object.entries(Settings.viewIncludes)) {
|
||||||
|
viewIncludes[view] = []
|
||||||
|
for (const filePath of paths) {
|
||||||
|
viewIncludes[view].push(
|
||||||
|
pug.compileFile(filePath, {
|
||||||
|
...PUG_COMPILE_ARGUMENTS,
|
||||||
|
cache: app.enabled('view cache'),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return viewIncludes
|
||||||
|
},
|
||||||
|
|
||||||
precompileViews(app) {
|
precompileViews(app) {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
let success = 0
|
let success = 0
|
||||||
|
let precompiled = 0
|
||||||
let failures = 0
|
let failures = 0
|
||||||
viewList.forEach(view => {
|
for (const filename of buildViewList()) {
|
||||||
const filename = path.resolve(view + '.pug') // express views are cached using the absolute path
|
const precompiledFilename = filename.replace(/\.pug$/, '.js')
|
||||||
|
if (fs.existsSync(precompiledFilename)) {
|
||||||
|
logger.debug({ filename }, 'loading precompiled pug template')
|
||||||
|
try {
|
||||||
|
pug.cache[filename] = require(precompiledFilename)
|
||||||
|
precompiled++
|
||||||
|
continue
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(
|
||||||
|
{ filename, err },
|
||||||
|
'error loading precompiled pug template'
|
||||||
|
)
|
||||||
|
failures++
|
||||||
|
}
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
pug.compileFile(filename, {
|
logger.warn({ filename }, 'compiling pug template at boot time')
|
||||||
cache: true,
|
pug.compileFile(filename, PUG_COMPILE_ARGUMENTS)
|
||||||
compileDebug: Settings.debugPugTemplates,
|
|
||||||
})
|
|
||||||
logger.debug({ filename }, 'compiled')
|
|
||||||
success++
|
success++
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error({ filename, err: err.message }, 'error compiling')
|
logger.error({ filename, err }, 'error compiling pug template')
|
||||||
failures++
|
failures++
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
logger.debug(
|
logger.debug(
|
||||||
{ timeTaken: Date.now() - startTime, failures, success },
|
{ timeTaken: Date.now() - startTime, failures, success, precompiled },
|
||||||
'compiled templates'
|
'compiled pug templates'
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
precompileViewsAndCacheToDisk()
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
|
@ -900,6 +900,7 @@ module.exports = {
|
||||||
'server-ce-scripts',
|
'server-ce-scripts',
|
||||||
'user-activate',
|
'user-activate',
|
||||||
],
|
],
|
||||||
|
viewIncludes: {},
|
||||||
|
|
||||||
csp: {
|
csp: {
|
||||||
enabled: process.env.CSP_ENABLED === 'true',
|
enabled: process.env.CSP_ENABLED === 'true',
|
||||||
|
|
|
@ -43,6 +43,7 @@
|
||||||
"routes": "bin/routes",
|
"routes": "bin/routes",
|
||||||
"storybook": "storybook dev -p 6006",
|
"storybook": "storybook dev -p 6006",
|
||||||
"build-storybook": "storybook build",
|
"build-storybook": "storybook build",
|
||||||
|
"precompile-pug": "node app/src/infrastructure/Views",
|
||||||
"local:nodemon": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; echo $OVERLEAF_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon",
|
"local:nodemon": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; echo $OVERLEAF_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon",
|
||||||
"local:webpack": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; PORT=13808 OVERLEAF_CONFIG=$(pwd)/config/settings.webpack.js npm run webpack",
|
"local:webpack": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; PORT=13808 OVERLEAF_CONFIG=$(pwd)/config/settings.webpack.js npm run webpack",
|
||||||
"local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",
|
"local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",
|
||||||
|
|
Loading…
Reference in a new issue