overleaf/services/web/bin/cdn_upload
Gernot Schulz 259663568f Merge pull request #15700 from overleaf/gs-verify-cdn-upload
Verify web CDN uploads

GitOrigin-RevId: d52b3b15ffa255cddd934bbaac4643dd48104428
2023-11-10 09:03:51 +00:00

64 lines
1.8 KiB
Bash
Executable file

#!/bin/bash
set -eEu
function upload_into_bucket() {
bucket=$1
# stylesheets
bin/cdn_upload_batch 'text/css' "$bucket" '.css' \
-x '.+(?<!\.css)$' &
# javascript files
bin/cdn_upload_batch 'application/javascript' "$bucket" '.js' \
-x '.+(?<!\.js)$' &
# the rest
bin/cdn_upload_batch '-' "$bucket" '-' \
-x '.+\.(css|js)$' &
wait
}
verify_upload_into_bucket() {
local bucket
local missing_from_bucket
bucket=$1
printf '\nINFO: Verifying file availability in %s.\n' "$bucket"
readarray -t missing_from_bucket < <(
comm -13 \
<(gsutil ls "${bucket}/public/**" | sed "s@${bucket}/@@" | sort) \
<(find /tmp/public /tmp/compressed -type f | sed '
# Remove absolute path prefix
s@^/tmp/@@;
# Undo the compressed/ directory separation that does not exist in the bucket
s@^compressed/@@
' | sort)
)
if [[ ${#missing_from_bucket[@]} -eq 0 ]]; then
printf 'INFO: Verification successful: all local files have been found in bucket %s.\n' \
"$bucket"
else
printf >&2 'WARN: %d local file(s) not available in bucket %s:\n' \
${#missing_from_bucket[@]} "$bucket"
printf >&2 ' - %s\n' "${missing_from_bucket[@]}"
return 1
fi
}
# Upload to staging CDN if branch is either 'main' or 'staging-main'
if [[ "$BRANCH_NAME" == "main" ]] || [[ "$BRANCH_NAME" == "staging-main" ]]; then
tar --directory=/tmp/ -xf build.tar
# delete source maps
find /tmp/public -name '*.js.map' -delete
bin/compress_assets
upload_into_bucket "$CDN_STAG" &&
verify_upload_into_bucket "$CDN_STAG" || exit 3
# Only upload to production CDN if branch is
if [[ "$BRANCH_NAME" == "main" ]]; then
upload_into_bucket "$CDN_PROD" &&
verify_upload_into_bucket "$CDN_PROD" || exit 3
fi
fi