bump versions

Dockerfile

@@ -41,6 +41,12 @@ RUN d="@$(date '+%s')" ; \
     find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
     find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
 
+WORKDIR /pycache
+RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \
+    | sort -zV \
+    | tar -C /usr/local --null -T - -cf - \
+    | tar -xf -
+
 ## ---
 
 FROM deps AS postgresql
@@ -66,7 +72,7 @@ RUN sed "s/%{PG_MAJOR}/${PG_MAJOR}/g" < /tmp/pgdg-ver.sources > "/etc/apt/source
     sed "s/%{PG_MAJOR}/${PG_MAJOR}/g" < /tmp/pgdg-ver.prefs   > "/etc/apt/preferences.d/pgdg-${PG_MAJOR}" ; \
     rm -f /tmp/pgdg-ver.sources /tmp/pgdg-ver.prefs
 
-COPY --from=postgresql-upstream /etc/postgresql-common/createcluster.conf  /etc/postgresql-common/
+COPY --from=postgresql-upstream  /etc/postgresql-common/createcluster.conf  /etc/postgresql-common/
 RUN apt-install.sh postgresql-common ; \
     apt-install.sh \
       "postgresql-${PG_MAJOR}" \
@@ -75,12 +81,12 @@ RUN apt-install.sh postgresql-common ; \
     f="/usr/share/postgresql/${PG_MAJOR}/postgresql.conf.sample" ; \
     dpkg-divert --add --rename --divert "$f.dpkg" "$f" ; \
     ln -sv ../postgresql.conf.sample "/usr/share/postgresql/${PG_MAJOR}/"
-COPY --from=postgresql-upstream /usr/share/postgresql/postgresql.conf.sample  /usr/share/postgresql/
+COPY --from=postgresql-upstream  /usr/share/postgresql/postgresql.conf.sample  /usr/share/postgresql/
 
 ENV PATH=${PATH}:/usr/lib/postgresql/${PG_MAJOR}/bin
 
-COPY --from=postgresql-upstream /docker-entrypoint-initdb.d/  /docker-entrypoint-initdb.d/
+COPY --from=postgresql-upstream  /docker-entrypoint-initdb.d/  /docker-entrypoint-initdb.d/
-COPY --from=postgresql-upstream /usr/local/bin/*.sh           /usr/local/bin/
+COPY --from=postgresql-upstream  /usr/local/bin/*.sh           /usr/local/bin/
 ## compatibility ;)
 RUN ln -sv /usr/local/sbin/dumb-run-as.sh /usr/local/bin/gosu
 
@@ -122,7 +128,8 @@ RUN apt-install.sh \
       "postgresql-${PG_MAJOR}-unit" \
       "postgresql-${PG_MAJOR}-wal2json" \
     ; \
-    apt-clean.sh
+    apt-clean.sh ; \
+    jdupes -1LSpr /usr/
 
 ## ---
 
@@ -134,7 +141,8 @@ COPY /apt/sources.citus  /etc/apt/sources.list.d/citus.sources
 RUN apt-install.sh \
       "postgresql-${PG_MAJOR}-citus-12.1" \
       "postgresql-${PG_MAJOR}-topn" \
-    ; apt-clean.sh
+    ; apt-clean.sh ; \
+    jdupes -1LSpr /usr/
 
 VOLUME [ "${PGHOME}" ]
 
@@ -146,8 +154,7 @@ SHELL [ "/bin/sh", "-ec" ]
 COPY /Dockerfile  /usr/local/share/
 
 ## RFC: Python cache
-## TODO: reduce load by selecting only __pycache__ directories in either way
+COPY --from=pycache  /pycache/  /usr/local/
-COPY --from=pycache /usr/local/lib/  /usr/local/lib/
 
 COPY /ep.sh             /usr/local/sbin/
 COPY /postgres-shim.sh  /usr/local/sbin/

Dockerfile.base

@@ -1,7 +1,7 @@
-ARG PYTHONTAG=3.11.9-slim-bookworm
+ARG PYTHONTAG=3.11.10-slim-bookworm
 FROM docker.io/python:${PYTHONTAG} AS base-upstream
 
-FROM base-upstream AS base
+FROM base-upstream AS base-intermediate
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile.base  /usr/local/share/
@@ -25,7 +25,8 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
 #     PIP_INDEX_URL="http://127.0.0.1:8081/repository/proxy_pypi/simple/" \
 #     PIP_TRUSTED_HOST="localhost"
 
-COPY /apt/sources.debian  /etc/apt/sources.list.d/debian.sources
+COPY /apt/preferences.backports  /etc/apt/preferences.d/backports
+COPY /apt/sources.debian         /etc/apt/sources.list.d/debian.sources
 
 ## prevent services from auto-starting, part 1
 RUN s='/usr/sbin/policy-rc.d' ; b='/usr/bin/policy-rc.d' ; \
@@ -45,7 +46,39 @@ RUN divert_true() { divert-rm.sh "$1" ; ln -sv /bin/true "$1" ; } ; \
     divert_true /usr/bin/deb-systemd-invoke
 
 RUN apt-env.sh apt-get update ; \
+    apt-remove.sh \
+      ca-certificates \
+      e2fsprogs \
+    ; \
     apt-env.sh apt-get upgrade -y ; \
+    apt-install.sh \
+      brotli \
+      cron \
+      curl \
+      dumb-init \
+      file \
+      gettext-base \
+      gnupg \
+      iproute2 \
+      iputils-ping \
+      jdupes \
+      jq \
+      less \
+      libnss-wrapper \
+      logrotate \
+      lsof \
+      ncurses-base \
+      netbase \
+      netcat-openbsd \
+      openssl \
+      procps \
+      psmisc \
+      tzdata \
+      vim \
+      xxd \
+      xz-utils \
+      zstd \
+    ; \
     apt-clean.sh
 
 ## perl-base: hardlink->symlink
@@ -74,6 +107,7 @@ RUN set -f ; \
       chsh \
       cpgr \
       cppw \
+      crontab \
       ctrlaltdel \
       debugfs \
       delpart \
@@ -206,35 +240,106 @@ RUN set -f ; \
         done ; \
     done
 
-RUN apt-remove.sh \
-      ca-certificates \
-      e2fsprogs \
-    ; \
-    apt-clean.sh
-
 ## "docker.io/python"-specific cleanup
 RUN rm -f /root/.wget-hsts
 
-RUN python-rm-cache.sh /usr/local ; \
+RUN pip-env.sh pip list --format freeze \
-    pip-env.sh pip list --format freeze \
     | grep -F '==' | awk -F= '{print $1}' \
     | xargs -r pip-env.sh pip install -U ; \
     python-rm-cache.sh "${PYTHON_SITE_PACKAGES}"
 
-RUN python-rm-cache.sh /usr/local ; \
+RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
-    libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
     rm -rfv \
       /usr/local/bin/idle* \
+      /usr/local/bin/pydoc* \
       "${libpython}/ensurepip/_bundled" \
       "${libpython}/idlelib" \
+      "${libpython}/pydoc.py" \
+      "${libpython}/pydoc_data" \
       "${libpython}/tkinter" \
       "${libpython}/turtle.py" \
       "${libpython}/turtledemo" \
-    ; :
+    ; \
+    rm -rfv \
+      "${PYTHON_SITE_PACKAGES}/pkg_resources/tests" \
+      "${PYTHON_SITE_PACKAGES}/setuptools/tests" \
+      "${PYTHON_SITE_PACKAGES}/setuptools/_distutils/tests" \
+      "${PYTHON_SITE_PACKAGES}/setuptools/_vendor/importlib_resources/tests" \
+    ; \
+    find "${PYTHON_SITE_PACKAGES}/" -iname '*.exe' -ls -delete ; \
+    python-rm-cache.sh /usr/local
+
+## adjust pip/certifi
+RUN certifi_pem="${PYTHON_SITE_PACKAGES}/pip/_vendor/certifi/cacert.pem" ; \
+    rm -f "${certifi_pem}" ; \
+    ln -s /etc/ssl/certs/ca-certificates.crt "${certifi_pem}"
 
 RUN find /usr/local/sbin/ ! -type d -ls -delete ; \
     find /run/ -mindepth 1 -ls -delete || : ; \
-    install -d -m 01777 /run/lock
+    install -d -m 01777 /run/lock ; \
+    jdupes -1LSpr /usr/
+
+## ---
+
+FROM base-intermediate AS certs
+SHELL [ "/bin/sh", "-ec" ]
+
+COPY /scripts/*        /usr/local/sbin/
+COPY /extra-scripts/*  /usr/local/sbin/
+
+## "2024.08.30"
+ENV CERTIFI_COMMIT=325c2fde4f8eec10d682b09f3b0414dc05e69a81
+
+# 'https://raw.githubusercontent.com/certifi/python-certifi'
+ARG CERTIFI_BASE_URI='https://github.com/certifi/python-certifi/raw'
+
+ARG CERTIFI_URI="${CERTIFI_BASE_URI}/${CERTIFI_COMMIT}/certifi/cacert.pem"
+ADD "${CERTIFI_URI}" /tmp/certifi.crt
+
+RUN apt-install.sh ca-certificates ; \
+    apt-clean.sh ; \
+    ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
+    ls -l "${ca_file}" ; \
+    ## process certifi
+    certifi-extras.sh /tmp/certifi.crt ; \
+    openssl-cert-auto-pem.sh "${ca_file}" "${ca_file}.new" "${ca_file}.fp" ; \
+    mv -f "${ca_file}.new" "${ca_file}" ; \
+    chmod 0444 "${ca_file}" "${ca_file}.fp" ; \
+    ls -l "${ca_file}" "${ca_file}.fp"
+
+## ---
+
+FROM base-intermediate AS apt-gpg
+SHELL [ "/bin/sh", "-ec" ]
+
+COPY /scripts/*        /usr/local/sbin/
+COPY /extra-scripts/*  /usr/local/sbin/
+
+COPY --from=certs  /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
+
+ADD https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc  /tmp/pgdg.gpg.bin
+ADD https://packagecloud.io/citusdata/community/gpgkey     /tmp/citus.gpg.bin
+
+## process GPG keyrings
+RUN pkg='gnupg' ; \
+    apt-install.sh ${pkg} ; \
+    gpg-export.sh /tmp/pgdg.gpg.bin /etc/apt/keyrings/pgdg.gpg.asc ; \
+    gpg-export.sh /tmp/citus.gpg.bin /etc/apt/keyrings/citus.gpg.asc ; \
+    apt-remove.sh ${pkg}
+
+COPY /apt/sources.pgdg   /etc/apt/sources.list.d/pgdg.sources
+COPY /apt/sources.citus  /etc/apt/sources.list.d/citus.sources
+
+## verify sources!
+RUN apt-env.sh apt-get update ; \
+    apt-clean.sh
+
+## ---
+
+FROM base-intermediate AS base
+
+COPY --from=certs    /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
+COPY --from=apt-gpg  /etc/apt/keyrings/                /etc/apt/keyrings/
 
 ENTRYPOINT [ ]
 CMD [ "bash" ]

Dockerfile.deps

@@ -1,52 +1,5 @@
-FROM docker.io/rockdrilla/postgresql:base-v1 AS base
+ARG BASE_IMAGE
-
+FROM ${BASE_IMAGE} AS base
-## ---
-
-FROM base AS certs
-SHELL [ "/bin/sh", "-ec" ]
-
-COPY /scripts/*        /usr/local/sbin/
-COPY /extra-scripts/*  /usr/local/sbin/
-
-## consult https://github.com/certifi/python-certifi/
-ENV CERTIFI_COMMIT=bd8153872e9c6fc98f4023df9c2deaffea2fa463
-
-RUN apt-install.sh ca-certificates curl ; \
-    apt-clean.sh ; \
-    ## process certifi
-    ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
-    ls -l "${ca_file}" ; \
-    certifi-extras.sh ; \
-    openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp" ; \
-    chmod 0444 "${ca_file}" "${ca_file}.fp" ; \
-    ls -l "${ca_file}" "${ca_file}.fp"
-
-## ---
-
-FROM base AS apt-gpg
-SHELL [ "/bin/sh", "-ec" ]
-
-COPY /scripts/*        /usr/local/sbin/
-COPY /extra-scripts/*  /usr/local/sbin/
-
-COPY --from=certs /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
-
-ADD https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc  /tmp/pgdg.gpg.bin
-ADD https://packagecloud.io/citusdata/community/gpgkey     /tmp/citus.gpg.bin
-
-## process GPG keyrings
-RUN pkg='gnupg' ; \
-    apt-install.sh ${pkg} ; \
-    gpg-export.sh /tmp/pgdg.gpg.bin /etc/apt/keyrings/pgdg.gpg.asc ; \
-    gpg-export.sh /tmp/citus.gpg.bin /etc/apt/keyrings/citus.gpg.asc ; \
-    apt-remove.sh ${pkg}
-
-COPY /apt/sources.pgdg   /etc/apt/sources.list.d/pgdg.sources
-COPY /apt/sources.citus  /etc/apt/sources.list.d/citus.sources
-
-## verify sources!
-RUN apt-env.sh apt-get update ; \
-    apt-clean.sh
 
 ## ---
 
@@ -55,21 +8,17 @@ SHELL [ "/bin/sh", "-ec" ]
 
 COPY /scripts/*  /usr/local/sbin/
 
-COPY --from=certs /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
-
-COPY --from=apt-gpg /etc/apt/keyrings/  /etc/apt/keyrings/
-
 COPY /apt/sources.pgdg      /etc/apt/sources.list.d/pgdg.sources
 COPY /apt/preferences.pgdg  /etc/apt/preferences.d/pgdg
 
+COPY /requirements.txt  /tmp/
+
 ENV DEV_PACKAGES='libffi-dev libpq-dev libyaml-dev'
 # psutil
 ENV CIBUILDWHEEL=1
 # pyyaml
 ENV PYYAML_FORCE_CYTHON=1
 
-COPY /requirements.txt  /tmp/
-
 RUN w=$(mktemp -d) ; : "${w:?}" ; \
     { apt-mark showauto ; apt-mark showmanual ; } | sort -uV > "$w/t0" ; \
     printf '%s\n' ${DEV_PACKAGES} | sort -uV > "$w/t1" ; \
@@ -88,39 +37,49 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \
       -r /tmp/requirements.txt \
     ; \
     pip-env.sh pip uninstall -y 'cython' ; \
-    python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" ; \
+    python-rm-cache.sh /usr/local ; \
     rm -rf \
       "${PYTHON_SITE_PACKAGES}/etcd/tests" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/tests" \
       "${PYTHON_SITE_PACKAGES}/psutil/tests" \
     ; \
+    truncate -s 0 \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/iab.idx" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/iab.txt" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/oui.txt" \
+      "${PYTHON_SITE_PACKAGES}/netaddr/eui/oui.idx" \
+    ; \
     find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \
     echo ; \
     find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -printf '%p\0' \
     | sed -zE '/rust/d' \
-    | xargs -0r strip --verbose --strip-debug ; \
+    | xargs -0r strip --verbose --strip-debug --strip-unneeded ; \
     echo ; \
     find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \
     apt-remove.sh build-essential ; \
     apt-clean.sh
 
+## avoid changing already present packages
+RUN find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
+    | sed -zEn \
+      -e '/^((pip|setuptools|wheel)-.+\.dist-info|distutils-precedence\.pth|_distutils_hack|pip|pkg_resources|setuptools|wheel)$/p' \
+    | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \
+      rm -rf
+
 ## ---
 
-FROM base
+FROM base AS deps
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile.deps  /usr/local/share/
 
 COPY /scripts/*  /usr/local/sbin/
 
-COPY --from=certs /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
-
-COPY --from=apt-gpg /etc/apt/keyrings/  /etc/apt/keyrings/
-
 COPY /apt/sources.pgdg      /etc/apt/sources.list.d/pgdg.sources
 COPY /apt/preferences.pgdg  /etc/apt/preferences.d/pgdg
 
-COPY --from=patroni /usr/local/bin/            /usr/local/bin/
+COPY --from=patroni  /usr/local/bin/            /usr/local/bin/
-COPY --from=patroni /${PYTHON_SITE_PACKAGES}/  /${PYTHON_SITE_PACKAGES}/
+COPY --from=patroni  /${PYTHON_SITE_PACKAGES}/  /${PYTHON_SITE_PACKAGES}/
 
 ## install missing dependencies for Python site-packages
 RUN f="${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \
@@ -128,33 +87,6 @@ RUN f="${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \
     xargs -a "$f" apt-install.sh ; \
     apt-clean.sh
 
-RUN apt-install.sh \
-      brotli \
-      curl \
-      dumb-init \
-      file \
-      gettext-base \
-      gnupg \
-      iproute2 \
-      iputils-ping \
-      jq \
-      less \
-      libnss-wrapper \
-      lsof \
-      ncurses-base \
-      netbase \
-      netcat-openbsd \
-      openssl \
-      procps \
-      psmisc \
-      tzdata \
-      vim \
-      xxd \
-      xz-utils \
-      zstd \
-    ; \
-    apt-clean.sh
-
 ## set up locales!
 RUN _lang=en_US.UTF8 ; \
     { \
@@ -174,4 +106,5 @@ RUN _lang=en_US.UTF8 ; \
     locale -a | grep -Fixq "${_lang}"
 ENV LANG=en_US.UTF8
 
-RUN find /usr/local/sbin/ ! -type d -ls -delete
+RUN find /usr/local/sbin/ ! -type d -ls -delete ; \
+    jdupes -1LSpr /usr/

apt/preferences.backports

@@ -0,0 +1,4 @@
+## example:
+# Package: src:curl
+# Pin: release n=bookworm-backports
+# Pin-Priority: 600

apt/sources.debian

@@ -1,11 +1,11 @@
 Types: deb
 URIs: http://deb.debian.org/debian
-Suites: bookworm bookworm-updates bookworm-proposed-updates
+Suites: bookworm bookworm-updates bookworm-proposed-updates bookworm-backports
-Components: main contrib non-free
+Components: main
 Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
 
 Types: deb
 URIs: http://deb.debian.org/debian-security
 Suites: bookworm-security
-Components: main contrib non-free
+Components: main
 Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg

build-scripts/image-base.sh

@@ -8,7 +8,7 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
 BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
 set +a
 
-PYTHONTAG="${PYTHONTAG:-3.11.9-slim-bookworm}"
+PYTHONTAG="${PYTHONTAG:-3.11.10-slim-bookworm}"
 
 grab_site_packages() {
 	podman run \
@@ -27,7 +27,7 @@ grab_site_packages() {
 PYTHON_SITE_PACKAGES=$(grab_site_packages "docker.io/python:${PYTHONTAG}")
 [ -n "${PYTHON_SITE_PACKAGES:?}" ]
 
-img="docker.io/rockdrilla/postgresql:base-v1"
+img="docker.io/rockdrilla/postgresql:base-v2"
 
 buildah bud \
   -f ./Dockerfile.base \

build-scripts/image-deps.sh

@@ -8,9 +8,11 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
 BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
 set +a
 
-img="docker.io/rockdrilla/postgresql:deps-v1"
+img="docker.io/rockdrilla/postgresql:deps-v2"
+base="docker.io/rockdrilla/postgresql:base-v2"
 
 exec buildah bud \
   -f ./Dockerfile.deps \
   -t "${img}" \
-  --pull=missing --no-cache
+  --pull=missing --no-cache \
+  --build-arg "BASE_IMAGE=${base}" \

build-scripts/image.sh

@@ -8,11 +8,11 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
 BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
 set +a
 
-POSTGRESQL_VERSION="${1:-16.4}"
+POSTGRESQL_VERSION="${1:-16.6}"
 PG_MAJOR="${POSTGRESQL_VERSION%%.*}"
 
 img="docker.io/rockdrilla/postgresql:${POSTGRESQL_VERSION}"
-deps="docker.io/rockdrilla/postgresql:deps-v1"
+deps="docker.io/rockdrilla/postgresql:deps-v2"
 
 c=$(buildah from --pull=missing "${deps}")
 [ -n "${c:?}" ]

extra-scripts/certifi-extras.sh

@@ -1,89 +1,41 @@
 #!/bin/sh
 set -ef
 
-certifi_uri="https://raw.githubusercontent.com/certifi/python-certifi/${CERTIFI_COMMIT:?}/certifi/cacert.pem"
 dst_dir=/usr/local/share/ca-certificates
 
 w=$(mktemp -d) ; : "${w:?}"
 w_cleanup() {
-	[ -z "$w" ] || ls -lA "$w/"
+	[ -z "$w" ] || ls -lA "$w/" >&2
 	[ -z "$w" ] || rm -rf "$w"
 	unset w
 	exit "${1:-0}"
 }
 
-curl -sSL "${certifi_uri}" > "$w/certifi.crt"
-
 def_bundle='/etc/ssl/certs/ca-certificates.crt'
 
-openssl-cert-auto-pem.sh "${def_bundle}"  > "$w/cacert.pem"
+openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp"
-openssl-cert-auto-pem.sh "$w/certifi.crt" > "$w/certifi.pem"
+[ -s "$w/cacert.pem" ] || w_cleanup 1
-[ -s "$w/cacert.pem" ]  || w_cleanup 1
+[ -s "$w/cacert.fp" ]  || w_cleanup 1
+
+openssl-cert-auto-pem.sh "$1" "$w/certifi.pem" "$w/certifi.fp" "$w/certifi.off"
 [ -s "$w/certifi.pem" ] || w_cleanup 1
-
+[ -s "$w/certifi.fp" ]  || w_cleanup 1
-bundle_offsets() {
-	awk '
-	BEGIN {
-	    OFS = ","
-	    m_begin="-----BEGIN CERTIFICATE-----"
-	    m_end="-----END CERTIFICATE-----"
-	    i_begin = 0
-	}
-	$0 == m_begin { i_begin = NR ; }
-	$0 == m_end {
-	    if (i_begin > 0) {
-	        print i_begin,NR
-	        i_begin = 0
-	    }
-	}
-	' "$1"
-}
-
-bundle_offsets "$w/cacert.pem"  > "$w/cacert.off"
-bundle_offsets "$w/certifi.pem" > "$w/certifi.off"
-[ -s "$w/cacert.off" ]  || w_cleanup 1
 [ -s "$w/certifi.off" ] || w_cleanup 1
 
-bundle_fingerprints() {
-	local a
-	while read -r a ; do
-		[ -n "$a" ] || continue
-
-		{
-			sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint -sha256 \
-			|| \
-			sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint
-		} | tr '[:upper:]' '[:lower:]'
-	done < "$2"
-}
-
-bundle_fingerprints "$w/cacert.pem"  "$w/cacert.off"  | sort -uV > "$w/cacert.fp"
-bundle_fingerprints "$w/certifi.pem" "$w/certifi.off" | sort -uV > "$w/certifi.fp"
-[ -s "$w/cacert.fp" ]  || w_cleanup 1
-[ -s "$w/certifi.fp" ] || w_cleanup 1
-
 set +e
-grep -Fxv -f "$w/cacert.fp" "$w/certifi.fp" > "$w/diff.fp"
+grep -Fxnv -f "$w/cacert.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/diff.ln"
 set -e
 
-if [ -s "$w/diff.fp" ] ; then
+if [ -s "$w/diff.ln" ] ; then
-	set +e
+	terse_fingerprint() { cut -d = -f 2- | tr -cd '[:alnum:]' ; }
-	grep -Fxn -f "$w/diff.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/records.diff"
-	set -e
-
-	terse_fingerprint() {
-		cut -d = -f 2- | tr -cd '[:alnum:]'
-	}
-
-	mkdir "$w/extras"
 
 	while read -r n ; do
 		[ -n "$n" ] || continue
 
 		fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint)
 		off=$(sed -ne "${n}p" "$w/certifi.off")
-		sed -ne "${off}p" "$w/certifi.pem" | openssl x509 > "${dst_dir}/certifi-${fp}.crt"
+		sed -ne "${off}p" "$w/certifi.pem" > "${dst_dir}/certifi-${fp}.crt"
-	done < "$w/records.diff"
+	done < "$w/diff.ln"
 fi
 
 rm -rf "$w" ; unset w

requirements.txt

@@ -1,27 +1,28 @@
 ## psycopg[c,pool]
 
-psycopg-c==3.2.1
+psycopg-c==3.2.3
 typing_extensions==4.12.2
-psycopg-pool==3.2.2
+psycopg-pool==3.2.4
-psycopg[c,pool]==3.2.1
+psycopg[c,pool]==3.2.3
 
 ## patroni[etcd3,kubernetes]
 
 click==8.1.7
 wcwidth==0.2.13
-prettytable==3.11.0
+prettytable==3.12.0
-psutil==6.0.0
+psutil==6.1.0
 six==1.16.0
 python-dateutil==2.9.0.post0
 PyYAML==6.0.2
-urllib3==2.2.2
+urllib3==2.2.3
-ydiff==1.3
+ydiff==1.4.2
-dnspython==2.6.1
+dnspython==2.7.0
 python-etcd==0.4.5
-patroni[etcd3,kubernetes]==3.3.2
+patroni[etcd3,kubernetes]==4.0.4
 
 ## misc
 cdiff==1.0
 pycparser==2.22
-cffi==1.17.0
+cffi==1.17.1
-cryptography==43.0.0
+cryptography==44.0.0
+netaddr==1.3.0

scripts/apt-clean.sh

@@ -23,24 +23,28 @@ find /var/cache/debconf/ ! -type d -wholename '/var/cache/debconf/*-old' -delete
 
 __t=$(mktemp) ; : "${__t:?}"
 debconf_trim_i18n() {
-    mawk 'BEGIN { m = 0 }
+	mawk 'BEGIN { m = 0; }
-    $0 == "" { print }
+	$0 == "" { print; }
-    /^[^[:space:]]/ {
+	/^[^[:space:]]/ {
-        if ($1 ~ "\.[Uu][Tt][Ff]-?8:") { m = 1; next; }
+	    if ($1 ~ "\.[Uu][Tt][Ff]-?8:") {
-        m = 0; print $0;
+	        m = 1;
-    }
+	        next;
-    /^[[:space:]]/ {
+	    }
-        if (m == 1) next;
+	    m = 0;
-        print $0;
+	    print $0;
-    }' < "$1" > "${__t}"
+	}
-    cat < "${__t}" > "$1"
+	/^[[:space:]]/ {
+	    if (m == 1) next;
+	    print $0;
+	}' < "$1" > "${__t}"
+	cat < "${__t}" > "$1"
 }
 
 debconf_trim_i18n /var/cache/debconf/templates.dat
 while read -r tmpl ; do
-    [ -n "${tmpl}" ] || continue
+	[ -n "${tmpl}" ] || continue
-    [ -s "${tmpl}" ] || continue
+	[ -s "${tmpl}" ] || continue
-    debconf_trim_i18n "${tmpl}"
+	debconf_trim_i18n "${tmpl}"
 done <<EOF
 $(find "${DPKG_ADMINDIR}/info/" -type f -name '*.templates' | sort -V)
 EOF

scripts/apt-install.sh

@@ -22,8 +22,9 @@ _apt_update() {
 }
 
 _dpkg_avail_hack() {
+	: "${DPKG_ADMINDIR:=/var/lib/dpkg}"
 	VERSION_CODENAME=$(. /etc/os-release ; printf '%s' "${VERSION_CODENAME}") || :
-	f="${DPKG_ADMINDIR:-/var/lib/dpkg}/available"
+	f="${DPKG_ADMINDIR}/available"
 	# if ${VERSION_CODENAME} is empty then we're on Debian sid or so :)
 	case "${VERSION_CODENAME}" in
 	stretch | buster | bionic | focal )
@@ -31,7 +32,7 @@ _dpkg_avail_hack() {
 		if [ -s "$f" ] ; then
 			return
 		fi
-		/usr/lib/dpkg/methods/apt/update "${DPKG_ADMINDIR:-/var/lib/dpkg}" apt apt
+		/usr/lib/dpkg/methods/apt/update "${DPKG_ADMINDIR}" apt apt
 	;;
 	* )
 		touch "$f"

scripts/openssl-cert-auto-pem.sh

@@ -13,38 +13,88 @@ me=${0##*/}
 
 w=$(mktemp -d) || exit 1
 w_cleanup() {
-	[ -z "$w" ] || ls -lA "$w/"
+	[ -z "$w" ] || ls -lA "$w/" >&2
 	[ -z "$w" ] || rm -rf "$w"
 	unset w
 	exit "${1:-0}"
 }
 
+bundle_offsets() {
+	mawk 'BEGIN { OFS = ","; i_begin = 0; }
+	$0 == "-----BEGIN CERTIFICATE-----" {
+	    i_begin = NR;
+	}
+	$0 == "-----END CERTIFICATE-----" {
+	    if (i_begin > 0) {
+	        print i_begin, NR;
+	        i_begin = 0;
+	    }
+	}' "$1"
+}
+
+bundle_fingerprints() {
+	local x f
+	while read -r x ; do
+		[ -n "$x" ] || continue
+
+		f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint -sha256)
+		[ -n "$f" ] || f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint)
+		[ -n "$f" ] || continue
+
+		printf '%s\n' "$f" | tr '[:upper:]' '[:lower:]'
+	done < "$2"
+}
+
 openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1
 [ -s "$w/cert.pem" ] || w_cleanup 1
 tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt"
 [ -s "$w/cert.txt" ] || w_cleanup 1
+rm -f "$w/cert.pem"
 
-awk '
+bundle_offsets "$w/cert.txt" > "$w/cert.off"
-BEGIN {
+[ -s "$w/cert.off" ] || w_cleanup 1
-    OFS = ","
-    m_begin="-----BEGIN CERTIFICATE-----"
-    m_end="-----END CERTIFICATE-----"
-    i_begin = 0
-}
-$0 == m_begin { i_begin = NR ; }
-$0 == m_end {
-    if (i_begin > 0) {
-        print i_begin,NR
-        i_begin = 0
-    }
-}
-' "$w/cert.txt" > "$w/cert.offsets"
-[ -s "$w/cert.offsets" ] || w_cleanup 1
 
-while read -r a ; do
+bundle_fingerprints "$w/cert.txt" "$w/cert.off" > "$w/cert.fp.all"
-	[ -n "$a" ] || continue
+[ -s "$w/cert.fp.all" ] || w_cleanup 1
 
-	sed -ne "${a}p" "$w/cert.txt"
+sort -uV < "$w/cert.fp.all" > "$w/cert.fp"
-done < "$w/cert.offsets"
+while read -r fp ; do
+	[ -n "${fp}" ] || continue
+
+	n=$(grep -m1 -Fxn -e "${fp}" "$w/cert.fp.all" | cut -d : -f 1)
+	[ -n "$n" ] || continue
+
+	off=$(sed -ne "${n}p" "$w/cert.off")
+	[ -n "${off}" ] || continue
+
+	sed -ne "${off}p" "$w/cert.txt"
+done < "$w/cert.fp" > "$w/cert.pem"
+[ -s "$w/cert.pem" ] || w_cleanup 1
+rm -f "$w/cert.txt" "$w/cert.off" "$w/cert.fp.all"
+
+if [ -n "$2" ] ; then
+	while : ; do
+		if [ -e "$2" ] ; then
+			[ -f "$2" ] || break
+		fi
+		cat > "$2"
+	break ; done
+else
+	cat
+fi < "$w/cert.pem"
+
+while [ -n "$3" ] ; do
+	if [ -e "$3" ] ; then
+		[ -f "$3" ] || break
+	fi
+	cat "$w/cert.fp" > "$3"
+break ; done
+
+while [ -n "$4" ] ; do
+	if [ -e "$4" ] ; then
+		[ -f "$4" ] || break
+	fi
+	bundle_offsets "$w/cert.pem" > "$4"
+break ; done
 
 rm -rf "$w" ; unset w

scripts/openssl-cert-fingerprint.sh

@@ -1,52 +0,0 @@
-#!/bin/sh
-set -f
-
-[ $# -gt 0 ] || exit 0
-me=${0##*/}
-
-[ -n "$1" ] || exit 1
-[ -f "$1" ] || {
-	env printf '%s: not a file or does not exist: %q\n' "${me}" "$1" >&2
-	exit 1
-}
-[ -s "$1" ] || exit 0
-
-w=$(mktemp -d) || exit 1
-w_cleanup() {
-	[ -z "$w" ] || ls -lA "$w/"
-	[ -z "$w" ] || rm -rf "$w"
-	unset w
-	exit "${1:-0}"
-}
-
-openssl-cert-auto-pem.sh "$1" > "$w/cert.pem" || w_cleanup 1
-[ -s "$w/cert.pem" ] || w_cleanup 1
-
-awk '
-BEGIN {
-    OFS = ","
-    m_begin="-----BEGIN CERTIFICATE-----"
-    m_end="-----END CERTIFICATE-----"
-    i_begin = 0
-}
-$0 == m_begin { i_begin = NR ; }
-$0 == m_end {
-    if (i_begin > 0) {
-        print i_begin,NR
-        i_begin = 0
-    }
-}
-' "$w/cert.pem" > "$w/cert.off"
-[ -s "$w/cert.off" ] || w_cleanup 1
-
-while read -r a ; do
-	[ -n "$a" ] || continue
-
-	{
-		sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint -sha256 \
-		|| \
-		sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint
-	} | tr '[:upper:]' '[:lower:]'
-done < "$w/cert.off"
-
-w_cleanup 0

scripts/python-rm-cache.sh

@@ -1,7 +1,9 @@
 #!/bin/sh
 set -f
 for i ; do
-    find "$i/" -name __pycache__ -exec rm -rf {} +
+	[ -n "$i" ] || continue
-    find "$i/" ! -type d -name '*.py[co]' -exec rm -f {} +
+	[ -d "$i" ] || continue
+	find "$i/" -name __pycache__ -exec rm -rf {} +
+	find "$i/" ! -type d -name '*.py[co]' -exec rm -f {} +
 done
 exit 0