From e30d11698082a3f85c74af601a1a179d9ed326eb Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 15 Jul 2021 16:54:38 +0100 Subject: [PATCH 1/2] re-add jemalloc profiling again Revert "Merge pull request #131 from overleaf/spd-revert-jemalloc" This reverts commit 5a80ecc0fed0fe8c653cabe36b0e379da729d5cc, reversing changes made to 6196386a61be1f38b1fbd5a651044b503ba0823c. --- services/git-bridge/Dockerfile | 25 +++++++++++++++++++++++-- services/git-bridge/start.sh | 2 ++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/services/git-bridge/Dockerfile b/services/git-bridge/Dockerfile index 04e74bc842..76545e1ef3 100644 --- a/services/git-bridge/Dockerfile +++ b/services/git-bridge/Dockerfile @@ -2,8 +2,26 @@ FROM maven:3-jdk-11 as base -RUN apt-get update && apt-get install -y make git sqlite3 \ - && rm -rf /var/lib/apt/lists +RUN apt-get update && apt-get install -y make git sqlite3 build-essential dpkg-dev + +RUN echo "deb-src http://deb.debian.org/debian buster main" >> /etc/apt/sources.list && \ + echo "deb-src http://deb.debian.org/debian buster-updates main" >> /etc/apt/sources.list && \ + echo "deb-src http://security.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +RUN mkdir -p /build +WORKDIR /build + +RUN apt-get update && \ + apt-get -y source libjemalloc-dev && \ + apt-get -y build-dep libjemalloc-dev && \ + echo "override_dh_auto_configure:" >> jemalloc-5.1.0/debian/rules && \ + echo "\tdh_auto_configure -- --enable-debug --enable-fill --enable-prof --enable-stat" >> jemalloc-5.1.0/debian/rules && \ + cat jemalloc-5.1.0/debian/rules + +WORKDIR /build/jemalloc-5.1.0 +RUN dpkg-buildpackage + +RUN rm -rf /var/lib/apt/lists COPY vendor/envsubst /opt/envsubst RUN chmod +x /opt/envsubst @@ -41,6 +59,9 @@ RUN mkdir /opt/cdbg && \ RUN useradd --create-home node COPY --from=builder /git-bridge.jar / +COPY --from=builder /build/*.deb /tmp/ + +RUN dpkg -i /tmp/libjemalloc*.deb COPY vendor/envsubst /opt/envsubst RUN chmod +x /opt/envsubst diff --git a/services/git-bridge/start.sh b/services/git-bridge/start.sh index 1aa0a9314a..fcd4642efe 100755 --- a/services/git-bridge/start.sh +++ b/services/git-bridge/start.sh @@ -16,4 +16,6 @@ if [ "$ENABLE_DEBUG_AGENT" == "true" ]; then GIT_BRIDGE_JVM_ARGS="-agentpath:/opt/cdbg/cdbg_java_agent.so -Dcom.google.cdbg.module=git-bridge -Dcom.google.cdbg.version=$VERSION ${GIT_BRIDGE_JVM_ARGS}" fi +export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so +export MALLOC_CONF=prof:true,lg_prof_interval:30,lg_prof_sample:17,prof_prefix:/tmp/jemalloc exec java $GIT_BRIDGE_JVM_ARGS -jar /git-bridge.jar /conf/runtime.json From e9232929c4fa7c7af8f8ac009c105ac644b3fa6e Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 15 Jul 2021 18:08:08 +0100 Subject: [PATCH 2/2] Add all dev/dbg packages to Docker image --- services/git-bridge/.dockerignore | 1 + services/git-bridge/Dockerfile | 19 +++++++++---- .../git-bridge/install-all-dev-packages.sh | 28 +++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100755 services/git-bridge/install-all-dev-packages.sh diff --git a/services/git-bridge/.dockerignore b/services/git-bridge/.dockerignore index acfeee7413..b1d7f66164 100644 --- a/services/git-bridge/.dockerignore +++ b/services/git-bridge/.dockerignore @@ -1,4 +1,5 @@ * +!install-all-dev-packages.sh !start.sh !/conf !/lib diff --git a/services/git-bridge/Dockerfile b/services/git-bridge/Dockerfile index 76545e1ef3..920b38caaa 100644 --- a/services/git-bridge/Dockerfile +++ b/services/git-bridge/Dockerfile @@ -21,6 +21,8 @@ RUN apt-get update && \ WORKDIR /build/jemalloc-5.1.0 RUN dpkg-buildpackage +ADD install-all-dev-packages.sh /install-all-dev-packages.sh + RUN rm -rf /var/lib/apt/lists COPY vendor/envsubst /opt/envsubst @@ -41,7 +43,10 @@ RUN make package \ -name 'writelatex-git-bridge*jar-with-dependencies.jar' \ -exec mv {} /git-bridge.jar \; -FROM openjdk:11-jre +FROM builder +# FROM openjdk:11-jre <-- disabled while we are memory profiling + +WORKDIR / RUN apt-get update && apt-get install -y git sqlite3 procps htop net-tools sockstat binutils graphviz \ && rm -rf /var/lib/apt/lists @@ -56,12 +61,16 @@ RUN mkdir /opt/cdbg && \ wget -qO- https://storage.googleapis.com/cloud-debugger/compute-java/debian-wheezy/cdbg_java_agent_gce.tar.gz | \ tar xvz -C /opt/cdbg -RUN useradd --create-home node +# Disabled while we are memory profiling (these are already in the image) +# RUN useradd --create-home node -COPY --from=builder /git-bridge.jar / -COPY --from=builder /build/*.deb /tmp/ +# COPY --from=builder /git-bridge.jar / +# COPY --from=builder /build/*.deb /tmp/ -RUN dpkg -i /tmp/libjemalloc*.deb +# RUN dpkg -i /tmp/libjemalloc*.deb +RUN dpkg -i /build/*.deb + +RUN apt-get -y update && /install-all-dev-packages.sh && rm -rf /var/lib/apt/lists COPY vendor/envsubst /opt/envsubst RUN chmod +x /opt/envsubst diff --git a/services/git-bridge/install-all-dev-packages.sh b/services/git-bridge/install-all-dev-packages.sh new file mode 100755 index 0000000000..bb01214f55 --- /dev/null +++ b/services/git-bridge/install-all-dev-packages.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +apt-cache pkgnames > /tmp/allpackages + +BADPKGLIST="libjemalloc2" +NEWPKGLIST="build-essential" + +echo "Searching for required -dev and -dbg packages..." +for PKG in `dpkg --get-selections | sed 's/[: ].*//'` +do + # Make sure it's not in the ignore list + echo $BADPKGLIST | grep -q $PKG + if [ $? -eq 0 ] + then + continue + fi + for suffix in dev dbg dbgsym + do + grep -qe "^$PKG-$suffix$" /tmp/allpackages + if [ $? -eq 0 ] + then + NEWPKGLIST=" $NEWPKGLIST $PKG-$suffix" + fi + done +done + +apt-get install -y $NEWPKGLIST +