Merge pull request #18559 from overleaf/jpa-server-ce-caching

[server-ce] docker image caching

GitOrigin-RevId: 161d775485d6b8f994d1a6e53440a586fb9a5058
This commit is contained in:
Jakob Ackermann 2024-05-30 09:13:56 +02:00 committed by Copybot
parent 5a1cff55bc
commit f5701b67eb
2 changed files with 66 additions and 26 deletions

View file

@ -1,24 +1,42 @@
# Makefile # Makefile
export OVERLEAF_BASE_TAG := sharelatex/sharelatex-base
export OVERLEAF_TAG := sharelatex/sharelatex
export OVERLEAF_BASE_CACHE := $(shell echo $(OVERLEAF_BASE_TAG) | sed -E 's/(.+):.+/\1:latest/')
export MONOREPO_REVISION := $(shell git rev-parse HEAD)
MONOREPO_ROOT := ../ MONOREPO_ROOT := ../
export MONOREPO_REVISION := $(shell git rev-parse HEAD)
export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
export OVERLEAF_BASE_BRANCH ?= sharelatex/sharelatex-base:$(BRANCH_NAME)
export OVERLEAF_BASE_LATEST ?= sharelatex/sharelatex-base
export OVERLEAF_BASE_TAG ?= sharelatex/sharelatex-base:$(BRANCH_NAME)-$(MONOREPO_REVISION)
export OVERLEAF_TAG_BRANCH ?= sharelatex/sharelatex:$(BRANCH_NAME)
export OVERLEAF_TAG_LATEST ?= sharelatex/sharelatex
export OVERLEAF_TAG ?= sharelatex/sharelatex:$(BRANCH_NAME)-$(MONOREPO_REVISION)
all: build-base build-community all: build-base build-community
build-base: build-base:
cp .dockerignore $(MONOREPO_ROOT) cp .dockerignore $(MONOREPO_ROOT)
ifeq ($(shell uname -m),x86_64) docker build \
docker pull $(OVERLEAF_BASE_CACHE) || echo "nothing cached yet" --build-arg BUILDKIT_INLINE_CACHE=1 \
endif --file Dockerfile-base \
docker build -f Dockerfile-base --pull --cache-from $(OVERLEAF_BASE_CACHE) -t $(OVERLEAF_BASE_TAG) $(MONOREPO_ROOT) --pull \
--cache-from $(OVERLEAF_BASE_LATEST) \
--cache-from $(OVERLEAF_BASE_BRANCH) \
--tag $(OVERLEAF_BASE_TAG) \
--tag $(OVERLEAF_BASE_BRANCH) \
$(MONOREPO_ROOT)
build-community: build-community:
cp .dockerignore $(MONOREPO_ROOT) cp .dockerignore $(MONOREPO_ROOT)
docker build --build-arg OVERLEAF_BASE_TAG --build-arg MONOREPO_REVISION -f Dockerfile -t $(OVERLEAF_TAG) $(MONOREPO_ROOT) docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg OVERLEAF_BASE_TAG \
--build-arg MONOREPO_REVISION \
--cache-from $(OVERLEAF_LATEST) \
--cache-from $(OVERLEAF_BRANCH) \
--file Dockerfile \
--tag $(OVERLEAF_TAG) \
--tag $(OVERLEAF_BRANCH) \
$(MONOREPO_ROOT)
.PHONY: all build-base build-community .PHONY: all build-base build-community

View file

@ -1,53 +1,75 @@
--- ---
steps: steps:
- id: build-base - id: build_base
name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder
dir: server-ce dir: server-ce
args: args:
- 'build-base' - 'build-base'
- 'OVERLEAF_BASE_CACHE=${_IMAGE_BASE_CACHE}'
- 'OVERLEAF_BASE_TAG=${_IMAGE_BASE}'
- id: build-community
name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder
dir: server-ce
args:
- build-community
- 'OVERLEAF_BASE_CACHE=${_IMAGE_BASE_CACHE}'
- 'OVERLEAF_BASE_TAG=${_IMAGE_BASE}'
- 'OVERLEAF_TAG=${_IMAGE_TAG}'
- id: tag_main_latest_base - id: tag_main_latest_base
name: gcr.io/cloud-builders/docker name: gcr.io/cloud-builders/docker
script: | script: |
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
if [ ${BRANCH_NAME} == main ]; then if [ ${BRANCH_NAME} == main ]; then
docker tag ${_IMAGE_BASE} ${_IMAGE_BASE_CACHE}; docker tag ${_IMAGE_BASE} ${_IMAGE_BASE_LATEST};
docker push ${_IMAGE_BASE_CACHE}; docker push ${_IMAGE_BASE_LATEST};
fi fi
automapSubstitutions: true automapSubstitutions: true
- id: prefetch_ce
name: gcr.io/cloud-builders/docker
script: |
#!/usr/bin/env bash
set +e # Ignore pull failures
docker pull ${_IMAGE_TAG_BRANCH}
docker pull ${_IMAGE_TAG_LATEST}
exit 0
automapSubstitutions: true
waitFor:
- '-'
- id: build_community
name: us-east1-docker.pkg.dev/overleaf-ops/ol-docker/cloud-builder
dir: server-ce
args:
- build-community
waitFor:
# do not wait for prefetch_ce, docker buildx will pull it as needed
- build_base
- id: tag_main_latest - id: tag_main_latest
name: gcr.io/cloud-builders/docker name: gcr.io/cloud-builders/docker
script: | script: |
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
if [ $BRANCH_NAME == main ]; then if [ $BRANCH_NAME == main ]; then
docker tag ${_IMAGE_TAG} ${_IMAGE_CE}; docker tag ${_IMAGE_TAG} ${_IMAGE_TAG_LATEST};
docker push ${_IMAGE_CE}; docker push ${_IMAGE_TAG_LATEST};
fi fi
automapSubstitutions: true automapSubstitutions: true
waitFor:
- build_community
timeout: 3600s timeout: 3600s
options: options:
machineType: E2_HIGHCPU_32 machineType: E2_HIGHCPU_32
env: env:
- 'BRANCH_NAME=${BRANCH_NAME}' - 'BRANCH_NAME=${BRANCH_NAME}'
# docker build
- 'OVERLEAF_BASE_BRANCH=${_IMAGE_BASE_BRANCH}'
- 'OVERLEAF_BASE_LATEST=${_IMAGE_BASE_LATEST}'
- 'OVERLEAF_BASE_TAG=${_IMAGE_BASE}'
- 'OVERLEAF_BRANCH=${_IMAGE_TAG_BRANCH}'
- 'OVERLEAF_LATEST=${_IMAGE_TAG_LATEST}'
- 'OVERLEAF_TAG=${_IMAGE_TAG}'
images: images:
- '${_IMAGE_BASE}' - '${_IMAGE_BASE}'
- '${_IMAGE_BASE_BRANCH}'
- '${_IMAGE_TAG}' - '${_IMAGE_TAG}'
- '${_IMAGE_TAG_BRANCH}'
substitutions: substitutions:
_IMAGE_BASE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}' _IMAGE_BASE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}'
_IMAGE_BASE_CACHE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:latest' _IMAGE_BASE_BRANCH: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:${BRANCH_NAME}'
_IMAGE_CE: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:latest' _IMAGE_BASE_LATEST: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base:latest'
_IMAGE_TAG_BRANCH: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:${BRANCH_NAME}'
_IMAGE_TAG_LATEST: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:latest'
_IMAGE_TAG: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}' _IMAGE_TAG: 'us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}'
tags: tags:
- 'overleaf-public' - 'overleaf-public'