refactor layout

python extensions are built using strictly versioned libpq-dev

Dockerfile

@@ -1,11 +1,11 @@
 ARG UPSTREAM_IMAGE_VERSION
-ARG DEPS_IMAGE
+ARG BASE_IMAGE
 FROM docker.io/library/postgres:${UPSTREAM_IMAGE_VERSION}-trixie AS postgresql-upstream
-FROM ${DEPS_IMAGE} AS deps
+FROM ${BASE_IMAGE} AS base
 
 ## ---
 
-FROM deps AS postgresql
+FROM base AS postgresql
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /scripts/*  /usr/local/sbin/
@@ -22,7 +22,9 @@ RUN echo "postgres:x:${PG_UID}:${PG_GID}:postgres:${PGHOME}:/bin/bash" >> /etc/p
 RUN install -d -o postgres -g postgres -m 3755 /run/postgresql
 VOLUME [ "/run/postgresql" ]
 
+COPY /apt/sources.pgdg             /etc/apt/sources.list.d/pgdg.sources
 COPY /apt/sources.pgdg-ver.in      /tmp/pgdg-ver.sources
+COPY /apt/preferences.pgdg         /etc/apt/preferences.d/pgdg
 COPY /apt/preferences.pgdg-ver.in  /tmp/pgdg-ver.prefs
 RUN sed "s/%{PG_MAJOR}/${PG_MAJOR}/g" < /tmp/pgdg-ver.sources > "/etc/apt/sources.list.d/pgdg-${PG_MAJOR}.sources" ; \
     sed "s/%{PG_MAJOR}/${PG_MAJOR}/g" < /tmp/pgdg-ver.prefs   > "/etc/apt/preferences.d/pgdg-${PG_MAJOR}" ; \
@@ -48,7 +50,135 @@ RUN ln -sv /usr/local/sbin/dumb-run-as.sh /usr/local/bin/gosu
 
 ## ---
 
-FROM postgresql AS postgresql-extras
+FROM postgresql AS pybuild
+SHELL [ "/bin/sh", "-ec" ]
+
+COPY /scripts/*  /usr/local/sbin/
+
+COPY /requirements.txt  /tmp/
+
+ENV DEV_PACKAGES='libffi-dev libpq-dev libyaml-dev'
+# psutil
+ENV CIBUILDWHEEL=1
+# pyyaml
+ENV PYYAML_FORCE_CYTHON=1
+
+RUN w=$(mktemp -d) ; : "${w:?}" ; \
+    { apt-mark showauto ; apt-mark showmanual ; } | sort -uV > "$w/t0" ; \
+    printf '%s\n' ${DEV_PACKAGES} | sort -uV > "$w/t1" ; \
+    apt-install.sh ${DEV_PACKAGES} ; \
+    { apt-mark showauto ; apt-mark showmanual ; } | sort -uV > "$w/t2" ; \
+    set +e ; \
+    grep -Fxv -f "$w/t0" "$w/t2" > "$w/t3" ; \
+    grep -Fxv -f "$w/t1" "$w/t3" > "$w/t4" ; \
+    grep -Ev -e '-(dev|doc)$' "$w/t4" > "${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \
+    set -e ; \
+    rm -rf "$w/" ; unset w ; \
+    apt-install.sh build-essential ; \
+    pip-env.sh pip install 'cython~=3.1.3' ; \
+    pip-env.sh pip install \
+      --no-binary 'cffi,psutil,pyyaml' \
+      -r /tmp/requirements.txt \
+    ; \
+    pip-env.sh pip uninstall -y 'cython' ; \
+    python-rm-cache.sh /usr/local ; \
+    rm -rf \
+      /usr/local/bin/patroni_aws \
+      /usr/local/bin/patroni_raft_controller \
+      "${PYTHON_SITE_PACKAGES}/etcd/tests" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/tests" \
+      "${PYTHON_SITE_PACKAGES}/psutil/tests" \
+    ; \
+    truncate -s 0 \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/iab.idx" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/iab.txt" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/oui.txt" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/oui.idx" \
+    ; \
+    find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \
+    echo ; \
+    find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -printf '%p\0' \
+    | sed -zE '/rust/d' \
+    | xargs -0r strip --strip-debug --strip-unneeded ; \
+    echo ; \
+    find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \
+    apt-remove.sh build-essential ; \
+    apt-clean.sh
+
+## avoid changing already present packages
+RUN rm -rfv \
+      /usr/local/bin/pip \
+      /usr/local/bin/pip3* \
+    ; \
+    find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
+    | sed -zEn \
+      -e '/^pip(|-.+\.dist-info)$/p' \
+    | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r rm -rf
+
+## ---
+
+FROM pybuild AS pycache
+SHELL [ "/bin/sh", "-ec" ]
+
+COPY /scripts/*  /usr/local/sbin/
+
+ENV PYTHONDONTWRITEBYTECODE=''
+
+## Python cache preseed
+
+RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
+    find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
+    | sed -zEn \
+      -e '/^(asyncio|collections|concurrent|ctypes|email|encodings|html|http|importlib|json|logging|multiprocessing|re|urllib|zipfile|zoneinfo)$/p' \
+    | sort -zV \
+    | env -C "${libpython}" xargs -0r \
+      python3 -m compileall -q -j 2 ; \
+    find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
+    | sed -zE \
+      -e '/\.(dist-info|pth|so|txt)$/d' \
+      -e '/^pip$/d' \
+    | sort -zV \
+    | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \
+      python3 -m compileall -q -j 2
+
+## Python cache warmup
+RUN export PYTHONPROFILEIMPORTTIME=1 ; \
+    patroni --help ; \
+    patronictl --help ; \
+    patroni_barman --help ; \
+    patroni_wale_restore --help ; \
+    cdiff --help ; \
+    netaddr --help ; \
+    ydiff --help
+
+## Python cache adjustments
+RUN d="@$(date '+%s')" ; \
+    find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
+    find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
+
+WORKDIR /pycache
+RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \
+    | sort -zV \
+    | tar -C /usr/local --null -T - -cf - \
+    | tar -xf -
+
+## ---
+
+FROM postgresql AS postgresql-patroni
+
+COPY --from=pybuild  /usr/local/bin/            /usr/local/bin/
+COPY --from=pybuild  /${PYTHON_SITE_PACKAGES}/  /${PYTHON_SITE_PACKAGES}/
+COPY --from=pycache  /pycache/                  /usr/local/
+
+## install missing dependencies for Python site-packages
+RUN f="${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \
+    [ -s "$f" ] || exit 0 ; \
+    xargs -a "$f" apt-install.sh ; \
+    apt-clean.sh
+
+## ---
+
+FROM postgresql-patroni AS postgresql-extras
 SHELL [ "/bin/sh", "-ec" ]
 
 RUN apt-install.sh \