Merge pull request #19764 from overleaf/jpa-server-pro-cache

[server-pro] faster local build

GitOrigin-RevId: d71abde37253e4c0398afa6935290af79a1204be

server-ce/Dockerfile

@@ -19,13 +19,14 @@ ADD services/ /overleaf/services/
 # -----------------------
 ADD patches/ /overleaf/patches
 
-# Install npm dependencies
+# Install npm dependencies and build webpack assets
 # ------------------------
-RUN node genScript install | bash
+RUN --mount=type=cache,target=/root/.cache \
-
+    --mount=type=cache,target=/root/.npm \
-# Compile
+    --mount=type=cache,target=/overleaf/services/web/node_modules/.cache,id=server-ce-webpack-cache \
-# --------------------
+    --mount=type=tmpfs,target=/tmp true \
-RUN node genScript compile | bash
+&&  node genScript install | bash \
+&&  node genScript compile | bash
 
 # Copy runit service startup scripts to its location
 # --------------------------------------------------

server-ce/Dockerfile-base

@@ -14,13 +14,20 @@ ENV REBUILT_AFTER="2024-07-09"
 
 # Install dependencies
 # --------------------
-RUN apt-get update \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+# Technically, we are using potentially stale package-lists with the below line.
+# Practically, apt refreshes the lists as needed and release builds run in fresh CI VMs without the cache.
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked true \
+# Enable caching: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages
+&&  rm -f /etc/apt/apt.conf.d/docker-clean && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+&&  apt-get update \
 &&  apt-get install -y \
       unattended-upgrades \
       build-essential wget net-tools unzip time imagemagick optipng strace nginx git python3 python-is-python3 zlib1g-dev libpcre3-dev gettext-base libwww-perl ca-certificates curl gnupg \
       qpdf \
       aspell aspell-en aspell-af aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-1901 aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-id aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-nr aspell-ns  aspell-pa aspell-pl aspell-pt aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-ss aspell-st aspell-sv aspell-tl aspell-tn aspell-ts aspell-uk aspell-uz aspell-xh aspell-zu \
-&&  unattended-upgrade -v \
+# upgrade base-image, batch all the upgrades together, rather than installing them on-by-one (which is slow!)
+&&  unattended-upgrade --verbose --no-minimal-upgrade-steps \
 # install Node.js https://github.com/nodesource/distributions#nodejs
 &&  mkdir -p /etc/apt/keyrings \
 &&  curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
@@ -31,8 +38,7 @@ RUN apt-get update \
 &&  rm -rf \
 # We are adding a custom nginx config in the main Dockerfile.
       /etc/nginx/nginx.conf \
-      /etc/nginx/sites-enabled/default \
+      /etc/nginx/sites-enabled/default
-      /var/lib/apt/lists/*
 
 # Install TexLive
 # ---------------

server-ce/Makefile

@@ -16,6 +16,7 @@ build-base:
 	cp .dockerignore $(MONOREPO_ROOT)
 	docker build \
 	  --build-arg BUILDKIT_INLINE_CACHE=1 \
+	  --progress=plain \
 	  --file Dockerfile-base \
 	  --pull \
 	  --cache-from $(OVERLEAF_BASE_LATEST) \
@@ -29,6 +30,7 @@ build-community:
 	cp .dockerignore $(MONOREPO_ROOT)
 	docker build \
 	  --build-arg BUILDKIT_INLINE_CACHE=1 \
+	  --progress=plain \
 	  --build-arg OVERLEAF_BASE_TAG \
 	  --build-arg MONOREPO_REVISION \
 	  --cache-from $(OVERLEAF_LATEST) \

server-ce/genScript.js

@@ -19,8 +19,6 @@ switch (process.argv.pop()) {
           console.log('npm install --include=dev')
           // run webpack
           console.log('npm run webpack:production')
-          // drop webpack/babel cache
-          console.log('rm -rf node_modules/.cache')
           // uninstall webpack and frontend dependencies
           console.log('npm install --omit=dev')
           // precompile pug
@@ -37,8 +35,3 @@ switch (process.argv.pop()) {
     console.log('exit 101')
     process.exit(101)
 }
-
-console.log('set +x')
-console.log(
-  'rm -rf /root/.cache /root/.npm $(find /tmp/ -mindepth 1 -maxdepth 1)'
-)

server-ce/test/package-lock.json

@@ -14,7 +14,7 @@
         "adm-zip": "^0.5.12",
         "body-parser": "^1.20.2",
         "celebrate": "^15.0.3",
-        "cypress": "^13.13.2",
+        "cypress": "13.13.2",
         "express": "^4.19.2",
         "isomorphic-git": "^1.25.10",
         "js-yaml": "^4.1.0",

services/web/frontend/macros/invalidate-babel-cache-if-needed.js

@@ -16,8 +16,15 @@ module.exports = function invalidateBabelCacheIfNeeded() {
     console.warn(
       'Detected change in overleafModuleImports, purging babel cache!'
     )
-    fs.rmSync(cachePath, { recursive: true, force: true, maxRetries: 5 })
+    // Gracefully handle cache mount in Server Pro build, only purge nested folders and keep .cache/ folder.
-    fs.mkdirSync(cachePath)
+    fs.mkdirSync(cachePath, { recursive: true })
+    for (const name of fs.readdirSync(cachePath)) {
+      fs.rmSync(Path.join(cachePath, name), {
+        recursive: true,
+        force: true,
+        maxRetries: 5,
+      })
+    }
     fs.writeFileSync(statePath, newState)
   }
 }