improve Python cache

Dockerfile

@@ -5,50 +5,6 @@ FROM ${DEPS_IMAGE} AS deps
 
 ## ---
 
-FROM deps AS pycache
-SHELL [ "/bin/sh", "-ec" ]
-
-COPY /scripts/*  /usr/local/sbin/
-
-ENV PYTHONDONTWRITEBYTECODE=''
-
-## Python cache preseed
-
-RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
-    find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
-    | sed -zEn \
-      -e '/^(collections|importlib|json|re)$/p' \
-    | sort -zV \
-    | env -C "${libpython}" xargs -0r \
-      python3 -m compileall -q -j 2 ; \
-    find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
-    | sed -zE \
-      -e '/\.(dist-info|pth|txt)$/d' \
-      -e '/^(pip|pkg_resources|setuptools|wheel)$/d' \
-    | sort -zV \
-    | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \
-      python3 -m compileall -q -j 2
-
-## Python cache warmup
-RUN export PYTHONPROFILEIMPORTTIME=1 ; \
-    patronictl --help ; \
-    patroni --help ; \
-    ydiff --help ; \
-    cdiff --help
-
-## Python cache adjustments
-RUN d="@$(date '+%s')" ; \
-    find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
-    find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
-
-WORKDIR /pycache
-RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \
-    | sort -zV \
-    | tar -C /usr/local --null -T - -cf - \
-    | tar -xf -
-
-## ---
-
 FROM deps AS postgresql
 SHELL [ "/bin/sh", "-ec" ]
 
@@ -138,8 +94,11 @@ SHELL [ "/bin/sh", "-ec" ]
 
 COPY /apt/sources.citus  /etc/apt/sources.list.d/citus.sources
 
-RUN apt-install.sh \
-      "postgresql-${PG_MAJOR}-citus-12.1" \
+RUN apt-env.sh apt-get update ; \
+    citus_pkg=$(apt-cache search "^postgresql-${PG_MAJOR}-citus-[0-9.]+\$" | awk '{print $1}' | sort -rV | head -n1) ; \
+    : "${citus_pkg:?}" ; \
+    apt-install.sh \
+      "${citus_pkg}" \
       "postgresql-${PG_MAJOR}-topn" \
     ; apt-clean.sh ; \
     jdupes -1LSpr /usr/
@@ -153,9 +112,6 @@ SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile  /usr/local/share/
 
-## RFC: Python cache
-COPY --from=pycache  /pycache/  /usr/local/
-
 COPY /ep.sh             /usr/local/sbin/
 COPY /postgres-shim.sh  /usr/local/sbin/
 ## quirk

Dockerfile.base

@@ -1,4 +1,4 @@
-ARG PYTHONTAG=3.11.11-slim-bookworm
+ARG PYTHONTAG=3.12.9-slim-bookworm
 FROM docker.io/python:${PYTHONTAG} AS base-upstream
 
 FROM base-upstream AS base-intermediate
@@ -241,7 +241,7 @@ RUN set -f ; \
     done
 
 ## "docker.io/python"-specific cleanup
-RUN rm -f /root/.wget-hsts
+RUN env -C /root rm -f .bash_history .python_history .wget-hsts
 
 RUN pip-env.sh pip list --format freeze \
     | grep -F '==' | awk -F= '{print $1}' \
@@ -260,12 +260,6 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
       "${libpython}/turtle.py" \
       "${libpython}/turtledemo" \
     ; \
-    rm -rfv \
-      "${PYTHON_SITE_PACKAGES}/pkg_resources/tests" \
-      "${PYTHON_SITE_PACKAGES}/setuptools/tests" \
-      "${PYTHON_SITE_PACKAGES}/setuptools/_distutils/tests" \
-      "${PYTHON_SITE_PACKAGES}/setuptools/_vendor/importlib_resources/tests" \
-    ; \
     find "${PYTHON_SITE_PACKAGES}/" -iname '*.exe' -ls -delete ; \
     python-rm-cache.sh /usr/local
 

Dockerfile.deps

@@ -39,6 +39,8 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \
     pip-env.sh pip uninstall -y 'cython' ; \
     python-rm-cache.sh /usr/local ; \
     rm -rf \
+      /usr/local/bin/patroni_aws \
+      /usr/local/bin/patroni_raft_controller \
       "${PYTHON_SITE_PACKAGES}/etcd/tests" \
       "${PYTHON_SITE_PACKAGES}/netaddr/tests" \
       "${PYTHON_SITE_PACKAGES}/psutil/tests" \
@@ -62,13 +64,13 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \
 ## avoid changing already present packages
 RUN find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
     | sed -zEn \
-      -e '/^((pip|setuptools|wheel)-.+\.dist-info|distutils-precedence\.pth|_distutils_hack|pip|pkg_resources|setuptools|wheel)$/p' \
+      -e '/^pip(|-.+\.dist-info)$/p' \
     | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \
       rm -rf
 
 ## ---
 
-FROM base AS deps
+FROM base AS deps-intermediate
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile.deps  /usr/local/share/
@@ -108,3 +110,55 @@ ENV LANG=en_US.UTF8
 
 RUN find /usr/local/sbin/ ! -type d -ls -delete ; \
     jdupes -1LSpr /usr/
+
+## ---
+
+FROM deps-intermediate AS pycache
+SHELL [ "/bin/sh", "-ec" ]
+
+COPY /scripts/*  /usr/local/sbin/
+
+ENV PYTHONDONTWRITEBYTECODE=''
+
+## Python cache preseed
+
+RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
+    find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
+    | sed -zEn \
+      -e '/^(asyncio|collections|concurrent|ctypes|email|encodings|html|http|importlib|json|logging|multiprocessing|re|urllib|zipfile|zoneinfo)$/p' \
+    | sort -zV \
+    | env -C "${libpython}" xargs -0r \
+      python3 -m compileall -q -j 2 ; \
+    find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
+    | sed -zE \
+      -e '/\.(dist-info|pth|so|txt)$/d' \
+      -e '/^pip$/d' \
+    | sort -zV \
+    | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \
+      python3 -m compileall -q -j 2
+
+## Python cache warmup
+RUN export PYTHONPROFILEIMPORTTIME=1 ; \
+    patroni --help ; \
+    patronictl --help ; \
+    patroni_barman --help ; \
+    patroni_wale_restore --help ; \
+    cdiff --help ; \
+    netaddr --help ; \
+    ydiff --help
+
+## Python cache adjustments
+RUN d="@$(date '+%s')" ; \
+    find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
+    find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
+
+WORKDIR /pycache
+RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \
+    | sort -zV \
+    | tar -C /usr/local --null -T - -cf - \
+    | tar -xf -
+  
+FROM deps-intermediate AS deps
+
+## RFC: Python cache
+COPY --from=pycache  /pycache/  /usr/local/

build-scripts/image-base.sh

@@ -8,7 +8,7 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
 BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
 set +a
 
-PYTHONTAG="${PYTHONTAG:-3.11.11-slim-bookworm}"
+PYTHONTAG="${PYTHONTAG:-3.12.9-slim-bookworm}"
 
 grab_site_packages() {
 	podman run \
@@ -27,7 +27,7 @@ grab_site_packages() {
 PYTHON_SITE_PACKAGES=$(grab_site_packages "docker.io/python:${PYTHONTAG}")
 [ -n "${PYTHON_SITE_PACKAGES:?}" ]
 
-img="docker.io/rockdrilla/postgresql:base-v2"
+img="docker.io/rockdrilla/postgresql:base-v3"
 
 buildah bud \
   -f ./Dockerfile.base \
@@ -36,10 +36,6 @@ buildah bud \
   --build-arg "PYTHONTAG=${PYTHONTAG}" \
   --env "PYTHON_SITE_PACKAGES=${PYTHON_SITE_PACKAGES}" \
   --unsetenv GPG_KEY \
-  --unsetenv PYTHON_PIP_VERSION \
-  --unsetenv PYTHON_SETUPTOOLS_VERSION \
-  --unsetenv PYTHON_GET_PIP_SHA256 \
-  --unsetenv PYTHON_GET_PIP_URL \
 
 
 c=$(buildah from --pull=never "${img}") || true

build-scripts/image-deps.sh

@@ -8,8 +8,8 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
 BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
 set +a
 
-img="docker.io/rockdrilla/postgresql:deps-v2"
-base="docker.io/rockdrilla/postgresql:base-v2"
+img="docker.io/rockdrilla/postgresql:deps-v3"
+base="docker.io/rockdrilla/postgresql:base-v3"
 
 exec buildah bud \
   -f ./Dockerfile.deps \

build-scripts/image.sh

@@ -12,7 +12,7 @@ POSTGRESQL_VERSION="${1:-16.7}"
 PG_MAJOR="${POSTGRESQL_VERSION%%.*}"
 
 img="docker.io/rockdrilla/postgresql:${POSTGRESQL_VERSION}"
-deps="docker.io/rockdrilla/postgresql:deps-v2"
+deps="docker.io/rockdrilla/postgresql:deps-v3"
 
 c=$(buildah from --pull=missing "${deps}")
 [ -n "${c:?}" ]

requirements.txt

@@ -18,7 +18,7 @@ urllib3==2.3.0
 ydiff==1.4.2
 dnspython==2.7.0
 python-etcd==0.4.5
-patroni[etcd3,kubernetes]==4.0.4
+patroni[etcd3,kubernetes]==4.0.5
 
 ## misc
 cdiff==1.0