From f5701b67eb804400dec2493b6a7f7915b0e91883 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 30 May 2024 09:13:56 +0200 Subject: [PATCH] Merge pull request #18559 from overleaf/jpa-server-ce-caching [server-ce] docker image caching GitOrigin-RevId: 161d775485d6b8f994d1a6e53440a586fb9a5058 --- server-ce/Makefile | 36 +++++++++++++++----- server-ce/cloudbuild.public.yaml | 56 ++++++++++++++++++++++---------- 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/server-ce/Makefile b/server-ce/Makefile index 723c0473b9..f32e63cf94 100644 --- a/server-ce/Makefile +++ b/server-ce/Makefile @@ -1,24 +1,42 @@ # Makefile -export OVERLEAF_BASE_TAG := sharelatex/sharelatex-base -export OVERLEAF_TAG := sharelatex/sharelatex -export OVERLEAF_BASE_CACHE := $(shell echo $(OVERLEAF_BASE_TAG) | sed -E 's/(.+):.+/\1:latest/') -export MONOREPO_REVISION := $(shell git rev-parse HEAD) MONOREPO_ROOT := ../ +export MONOREPO_REVISION := $(shell git rev-parse HEAD) +export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +export OVERLEAF_BASE_BRANCH ?= sharelatex/sharelatex-base:$(BRANCH_NAME) +export OVERLEAF_BASE_LATEST ?= sharelatex/sharelatex-base +export OVERLEAF_BASE_TAG ?= sharelatex/sharelatex-base:$(BRANCH_NAME)-$(MONOREPO_REVISION) +export OVERLEAF_TAG_BRANCH ?= sharelatex/sharelatex:$(BRANCH_NAME) +export OVERLEAF_TAG_LATEST ?= sharelatex/sharelatex +export OVERLEAF_TAG ?= sharelatex/sharelatex:$(BRANCH_NAME)-$(MONOREPO_REVISION) all: build-base build-community build-base: cp .dockerignore $(MONOREPO_ROOT) -ifeq ($(shell uname -m),x86_64) - docker pull $(OVERLEAF_BASE_CACHE) || echo "nothing cached yet" -endif - docker build -f Dockerfile-base --pull --cache-from $(OVERLEAF_BASE_CACHE) -t $(OVERLEAF_BASE_TAG) $(MONOREPO_ROOT) + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --file Dockerfile-base \ + --pull \ + --cache-from $(OVERLEAF_BASE_LATEST) \ + --cache-from $(OVERLEAF_BASE_BRANCH) \ + --tag $(OVERLEAF_BASE_TAG) \ + --tag $(OVERLEAF_BASE_BRANCH) \ + $(MONOREPO_ROOT) build-community: cp .dockerignore $(MONOREPO_ROOT) - docker build --build-arg OVERLEAF_BASE_TAG --build-arg MONOREPO_REVISION -f Dockerfile -t $(OVERLEAF_TAG) $(MONOREPO_ROOT) + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg OVERLEAF_BASE_TAG \ + --build-arg MONOREPO_REVISION \ + --cache-from $(OVERLEAF_LATEST) \ + --cache-from $(OVERLEAF_BRANCH) \ + --file Dockerfile \ + --tag $(OVERLEAF_TAG) \ + --tag $(OVERLEAF_BRANCH) \ + $(MONOREPO_ROOT) .PHONY: all build-base build-community diff --git a/server-ce/cloudbuild.public.yaml b/server-ce/cloudbuild.public.yaml index 2a0ec081b3..f682a173e6 100644 --- a/server-ce/cloudbuild.public.yaml +++ b/server-ce/cloudbuild.public.yaml @@ -1,53 +1,75 @@ --- steps: - - id: build-base + - id: build_base name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder dir: server-ce args: - 'build-base' - - 'OVERLEAF_BASE_CACHE=${_IMAGE_BASE_CACHE}' - - 'OVERLEAF_BASE_TAG=${_IMAGE_BASE}' - - id: build-community - name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder - dir: server-ce - args: - - build-community - - 'OVERLEAF_BASE_CACHE=${_IMAGE_BASE_CACHE}' - - 'OVERLEAF_BASE_TAG=${_IMAGE_BASE}' - - 'OVERLEAF_TAG=${_IMAGE_TAG}' - id: tag_main_latest_base name: gcr.io/cloud-builders/docker script: | #!/usr/bin/env bash set -e if [ ${BRANCH_NAME} == main ]; then - docker tag ${_IMAGE_BASE} ${_IMAGE_BASE_CACHE}; - docker push ${_IMAGE_BASE_CACHE}; + docker tag ${_IMAGE_BASE} ${_IMAGE_BASE_LATEST}; + docker push ${_IMAGE_BASE_LATEST}; fi automapSubstitutions: true + - id: prefetch_ce + name: gcr.io/cloud-builders/docker + script: | + #!/usr/bin/env bash + set +e # Ignore pull failures + docker pull ${_IMAGE_TAG_BRANCH} + docker pull ${_IMAGE_TAG_LATEST} + exit 0 + automapSubstitutions: true + waitFor: + - '-' + - id: build_community + name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder + dir: server-ce + args: + - build-community + waitFor: + # do not wait for prefetch_ce, docker buildx will pull it as needed + - build_base - id: tag_main_latest name: gcr.io/cloud-builders/docker script: | #!/usr/bin/env bash set -e if [ $BRANCH_NAME == main ]; then - docker tag ${_IMAGE_TAG} ${_IMAGE_CE}; - docker push ${_IMAGE_CE}; + docker tag ${_IMAGE_TAG} ${_IMAGE_TAG_LATEST}; + docker push ${_IMAGE_TAG_LATEST}; fi automapSubstitutions: true + waitFor: + - build_community timeout: 3600s options: machineType: E2_HIGHCPU_32 env: - 'BRANCH_NAME=${BRANCH_NAME}' + # docker build + - 'OVERLEAF_BASE_BRANCH=${_IMAGE_BASE_BRANCH}' + - 'OVERLEAF_BASE_LATEST=${_IMAGE_BASE_LATEST}' + - 'OVERLEAF_BASE_TAG=${_IMAGE_BASE}' + - 'OVERLEAF_BRANCH=${_IMAGE_TAG_BRANCH}' + - 'OVERLEAF_LATEST=${_IMAGE_TAG_LATEST}' + - 'OVERLEAF_TAG=${_IMAGE_TAG}' images: - '${_IMAGE_BASE}' + - '${_IMAGE_BASE_BRANCH}' - '${_IMAGE_TAG}' + - '${_IMAGE_TAG_BRANCH}' substitutions: _IMAGE_BASE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}' - _IMAGE_BASE_CACHE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:latest' - _IMAGE_CE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:latest' + _IMAGE_BASE_BRANCH: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:${BRANCH_NAME}' + _IMAGE_BASE_LATEST: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:latest' + _IMAGE_TAG_BRANCH: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:${BRANCH_NAME}' + _IMAGE_TAG_LATEST: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:latest' _IMAGE_TAG: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}' tags: - 'overleaf-public'