diff --git a/Dockerfile b/Dockerfile index c43502e..799673f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,8 +84,7 @@ RUN apt-install.sh \ "postgresql-${PG_MAJOR}-unit" \ "postgresql-${PG_MAJOR}-wal2json" \ ; \ - apt-clean.sh ; \ - jdupes -1LSpr /usr/ + apt-clean.sh ## --- @@ -100,8 +99,7 @@ RUN apt-env.sh apt-get update ; \ apt-install.sh \ "${citus_pkg}" \ "postgresql-${PG_MAJOR}-topn" \ - ; apt-clean.sh ; \ - jdupes -1LSpr /usr/ + ; apt-clean.sh VOLUME [ "${PGHOME}" ] @@ -123,8 +121,7 @@ WORKDIR "${PGHOME}" ## NB: override to SIGTERM in order to switch to "Smart Shutdown mode" STOPSIGNAL SIGINT -ENV DUMB_INIT_SETSID=0 \ - MALLOC_ARENA_MAX=4 \ +ENV MALLOC_ARENA_MAX=4 \ GOMAXPROCS=4 ENTRYPOINT [ "ep.sh" ] diff --git a/Dockerfile.base b/Dockerfile.base index 518cf83..2906453 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -20,11 +20,6 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 -## local development -# ENV PIP_INDEX="http://127.0.0.1:8081/repository/proxy_pypi/pypi/" \ -# PIP_INDEX_URL="http://127.0.0.1:8081/repository/proxy_pypi/simple/" \ -# PIP_TRUSTED_HOST="localhost" - COPY /apt/preferences.backports /etc/apt/preferences.d/backports COPY /apt/sources.debian /etc/apt/sources.list.d/debian.sources @@ -56,7 +51,6 @@ RUN apt-env.sh apt-get update ; \ brotli \ cron \ curl \ - dumb-init \ file \ gettext-base \ gnupg \ @@ -65,6 +59,8 @@ RUN apt-env.sh apt-get update ; \ jdupes \ jq \ less \ + libcap2-bin \ + libjemalloc2 \ libnss-wrapper \ logrotate \ lsof \ @@ -83,9 +79,20 @@ RUN apt-env.sh apt-get update ; \ apt-clean.sh ## perl-base: hardlink->symlink -RUN d=/usr/bin ; \ - find "$d/" -wholename "$d/perl5*" -exec ln -fsv perl {} ';' ; \ - ls -li "$d/perl"* +RUN set +e ; \ + d=/usr/bin ; \ + ls -li "$d/perl" ; \ + find "$d/" -xdev -samefile "$d/perl" 2>/dev/null \ + | grep -Fxv -e "$d/perl" \ + | while read -r p ; do \ + [ -n "$p" ] || continue ; \ + [ -e "$p" ] || continue ; \ + ls -li "$p" ; \ + rm -fv "$p" ; \ + ln -fsv perl "$p" ; \ + ls -li "$p" ; \ + echo ; \ + done ## remove unwanted binaries RUN set -f ; \ @@ -195,22 +202,30 @@ RUN set -f ; \ 'write.*' \ zramctl \ ; do \ - for d in /usr/sbin /usr/bin /sbin /bin ; do \ - find "$d/" ! -type d -wholename "$d/$i" \ - | while read -r p ; do \ - [ -n "$p" ] || continue ; \ - [ -e "$p" ] || continue ; \ - dpkg -S "$p" >/dev/null 2>&1 || continue ; \ - divert-rm.sh "$p" ; \ - done ; \ + ## try dpkg-divert first + for d in /usr/sbin /usr/bin ; do \ + find "$d/" ! -type d -wholename "$d/$i" ; \ + done \ + | while read -r p ; do \ + [ -n "$p" ] || continue ; \ + [ -e "$p" ] || continue ; \ + dpkg-search.sh "$p" || continue ; \ + done \ + | sed -E '/^diversion by/d' \ + | sort -uV \ + | while read -r pkg path ; do \ + [ -n "${pkg}" ] || continue ; \ + [ -e "${path}" ] || continue ; \ + divert-rm.sh "${path}" ; \ done ; \ - for d in /usr/sbin /usr/bin /sbin /bin ; do \ - find "$d/" ! -type d -wholename "$d/$i" \ - | while read -r p ; do \ - [ -n "$p" ] || continue ; \ - [ -e "$p" ] || continue ; \ - rm -fv "$p" ; \ - done ; \ + ## remove if still exists + for d in /usr/sbin /usr/bin ; do \ + find "$d/" ! -type d -wholename "$d/$i" ; \ + done \ + | while read -r p ; do \ + [ -n "$p" ] || continue ; \ + [ -e "$p" ] || continue ; \ + rm -fv "$p" ; \ done ; \ done ; \ ## fixup @@ -220,60 +235,42 @@ RUN set -f ; \ /sbin/getty \ ; : -## remove excessive privileges from binaries -RUN set -f ; \ - for i in \ - passwd \ - su \ - ; do \ - for d in /usr/sbin /usr/bin /sbin /bin ; do \ - find "$d/" ! -type d -wholename "$d/$i" \ - | while read -r p ; do \ - [ -n "$p" ] || continue ; \ - [ -e "$p" ] || continue ; \ - dpkg -S "$p" >/dev/null 2>&1 || continue ; \ - o=$(env stat -c '%U' "$p") ; \ - g=$(env stat -c '%G' "$p") ; \ - ls -l "$p" ; \ - dpkg-statoverride --update --add "$o" "$g" 0755 "$p" ; \ - ls -l "$p" ; \ - done ; \ - done ; \ +## remove excessive privileges from binaries: setuid/setgid +RUN find / -xdev -type f -perm /7000 \ + | sort -V \ + | while read -r p ; do \ + [ -n "$p" ] || continue ; \ + ## clear setuid/setgid bit + m=$(env stat -c '0%a' "$p") ; \ + m=$(printf '0%o\n' $((m & 00777)) ) ; \ + ## try to lookup in dpkg database + n=$(set +e ; dpkg-search.sh "$p" | sed -E '/^diversion by/d' | cut -d ' ' -f2-) ; \ + ls -l "$p" ; \ + if [ "$p" = "$n" ] ; then \ + o=$(env stat -c '%U' "$n") ; \ + g=$(env stat -c '%G' "$n") ; \ + dpkg-statoverride --force --update --add "$o" "$g" "$m" "$n" ; \ + else \ + env printf 'unable to find in dpkg database: %q\n' "$n" ; \ + chmod "$m" "$p" ; \ + fi ; \ + ls -l "$p" ; \ + done + +## remove excessive privileges from binaries: setcap +RUN find / -xdev -type f -executable -exec getcap {} + \ + | sort -V \ + | while read -r path caps ; do \ + [ -n "${path}" ] || continue ; \ + if [ "${path}" = /usr/bin/ping ] ; then continue ; fi ; \ + getcap -v "${path}" ; \ + setcap -r "${path}" "${caps}" 2>/dev/null || : ; \ + getcap -v "${path}" ; \ done ## "docker.io/python"-specific cleanup RUN env -C /root rm -f .bash_history .python_history .wget-hsts -RUN pip-env.sh pip list --format freeze \ - | grep -F '==' | awk -F= '{print $1}' \ - | xargs -r pip-env.sh pip install -U ; \ - python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" - -RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ - rm -rfv \ - /usr/local/bin/idle* \ - /usr/local/bin/pydoc* \ - "${libpython}/ensurepip/_bundled" \ - "${libpython}/idlelib" \ - "${libpython}/pydoc.py" \ - "${libpython}/pydoc_data" \ - "${libpython}/tkinter" \ - "${libpython}/turtle.py" \ - "${libpython}/turtledemo" \ - ; \ - find "${PYTHON_SITE_PACKAGES}/" -iname '*.exe' -ls -delete ; \ - python-rm-cache.sh /usr/local - -## adjust pip/certifi -RUN certifi_pem="${PYTHON_SITE_PACKAGES}/pip/_vendor/certifi/cacert.pem" ; \ - rm -f "${certifi_pem}" ; \ - ln -s /etc/ssl/certs/ca-certificates.crt "${certifi_pem}" - -RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ - find /run/ -mindepth 1 -ls -delete || : ; \ - install -d -m 01777 /run/lock ; \ - jdupes -1LSpr /usr/ - ## --- FROM base-intermediate AS certs @@ -336,5 +333,40 @@ FROM base-intermediate AS base COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/ +RUN python-rm-cache.sh /usr/local + +RUN pip-env.sh pip list --format freeze \ + | grep -F '==' | awk -F= '{print $1}' \ + | xargs -r pip-env.sh pip install -U ; \ + python-rm-cache.sh /usr/local + +RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ + rm -rfv \ + /usr/local/bin/idle* \ + /usr/local/bin/pydoc* \ + "${libpython}/ensurepip/_bundled" \ + "${libpython}/idlelib" \ + "${libpython}/pydoc.py" \ + "${libpython}/pydoc_data" \ + "${libpython}/tkinter" \ + "${libpython}/turtle.py" \ + "${libpython}/turtledemo" \ + ; \ + find "${PYTHON_SITE_PACKAGES}/" -iname '*.exe' -ls -delete + +## adjust pip/certifi +RUN certifi_pem="${PYTHON_SITE_PACKAGES}/pip/_vendor/certifi/cacert.pem" ; \ + [ -d "${certifi_pem%/*}" ] || exit 0 ; \ + rm -f "${certifi_pem}" ; \ + ln -sv /etc/ssl/certs/ca-certificates.crt "${certifi_pem}" + +RUN apt-clean.sh + +RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ + find /run/ -mindepth 1 -ls -delete || : ; \ + install -d -m 01777 /run/lock ; \ + jdupes -1LSpr /usr/local/ ; \ + jdupes -1LSpr /usr/ + ENTRYPOINT [ ] CMD [ "bash" ] diff --git a/Dockerfile.deps b/Dockerfile.deps index a939596..c00ea1f 100644 --- a/Dockerfile.deps +++ b/Dockerfile.deps @@ -3,6 +3,58 @@ FROM ${BASE_IMAGE} AS base ## --- +FROM base AS catatonit +SHELL [ "/bin/sh", "-ec" ] + +COPY /scripts/* /usr/local/sbin/ +COPY /extra-scripts/* /usr/local/sbin/ + +## current HEAD: "main: don't use secure_getenv", December 14, 2024 +ENV CATATONIT_COMMIT=56579adbb42c0c7ad94fc12d844b38fc5b37b3ce + +# ARG CATATONIT_BASE_URI='https://codeload.github.com/openSUSE/catatonit/tar.gz' +# ARG CATATONIT_URI="${CATATONIT_BASE_URI}/${CATATONIT_COMMIT}" + +ARG CATATONIT_BASE_URI='https://github.com/openSUSE/catatonit/archive' +ARG CATATONIT_URI="${CATATONIT_BASE_URI}/${CATATONIT_COMMIT}.tar.gz" + +# hadolint ignore=DL3020 +ADD "${CATATONIT_URI}" /tmp/catatonit.tar.gz + +RUN pkg='build-essential debhelper musl-dev autoconf autoconf-archive' ; \ + apt-install.sh ${pkg} ; \ + ## build catatonit + d=/tmp/catatonit ; \ + rm -rf "$d" ; \ + mkdir -p "$d" ; \ + ( \ + cd "$d" ; \ + tar --strip-components=1 -xf /tmp/catatonit.tar.gz ; \ + commit_abbrev=$(printf '%s' "${CATATONIT_COMMIT}" | cut -c1-8) ; \ + sed -i "s/+dev/+git.${commit_abbrev}/" configure.ac ; \ + # DEB_HOST_GNU_TYPE=$(dpkg-architecture -q DEB_HOST_GNU_TYPE) ; \ + # export HOSTCC="${DEB_HOST_GNU_TYPE}-gcc" ; \ + DEB_TARGET_GNU_TYPE=$(dpkg-architecture -q DEB_TARGET_GNU_TYPE) ; \ + DEB_TARGET_MUSL_TYPE=$(printf '%s' "${DEB_TARGET_GNU_TYPE}" | sed -E 's/-gnu$/-musl/') ; \ + export CC="${DEB_TARGET_MUSL_TYPE}-gcc" ; \ + export CFLAGS='-Os -g -pipe -fpie -fstack-protector-strong -fstack-clash-protection -fcf-protection -D_FORTIFY_SOURCE=2' ; \ + export LDFLAGS='-static-pie -Wl,-z -Wl,relro' ; \ + autoreconf -fiv ; \ + ./configure ; \ + make -j1 ; \ + ls -l catatonit ; \ + # "${DEB_TARGET_GNU_TYPE}-strip" --strip-debug --strip-unneeded catatonit ; \ + strip --strip-debug --strip-unneeded catatonit ; \ + ls -l catatonit ; \ + cp catatonit /usr/local/bin/ ; \ + ) ; \ + ## cleanup + rm -rf "$d" ; \ + apt-remove.sh ${pkg} ; \ + apt-clean.sh + +## --- + FROM base AS patroni SHELL [ "/bin/sh", "-ec" ] @@ -31,7 +83,7 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \ set -e ; \ rm -rf "$w/" ; unset w ; \ apt-install.sh build-essential ; \ - pip-env.sh pip install 'cython' ; \ + pip-env.sh pip install 'cython~=3.0.12' ; \ pip-env.sh pip install \ --no-binary 'cffi,psutil,pyyaml' \ -r /tmp/requirements.txt \ @@ -55,18 +107,21 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \ echo ; \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -printf '%p\0' \ | sed -zE '/rust/d' \ - | xargs -0r strip --verbose --strip-debug --strip-unneeded ; \ + | xargs -0r strip --strip-debug --strip-unneeded ; \ echo ; \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \ apt-remove.sh build-essential ; \ apt-clean.sh ## avoid changing already present packages -RUN find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \ +RUN rm -rfv \ + /usr/local/bin/pip \ + /usr/local/bin/pip3* \ + ; \ + find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \ | sed -zEn \ -e '/^pip(|-.+\.dist-info)$/p' \ - | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \ - rm -rf + | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r rm -rf ## --- @@ -80,6 +135,8 @@ COPY /scripts/* /usr/local/sbin/ COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg +COPY --from=catatonit /usr/local/bin/catatonit /usr/local/bin/ + COPY --from=patroni /usr/local/bin/ /usr/local/bin/ COPY --from=patroni /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/ @@ -108,8 +165,7 @@ RUN _lang=en_US.UTF8 ; \ locale -a | grep -Fixq "${_lang}" ENV LANG=en_US.UTF8 -RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ - jdupes -1LSpr /usr/ +RUN find /usr/local/sbin/ ! -type d -ls -delete ## --- @@ -157,7 +213,9 @@ RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \ | sort -zV \ | tar -C /usr/local --null -T - -cf - \ | tar -xf - - + +## --- + FROM deps-intermediate AS deps ## RFC: Python cache diff --git a/apt/preferences.backports b/apt/preferences.backports index dabc1ee..fe469f4 100644 --- a/apt/preferences.backports +++ b/apt/preferences.backports @@ -2,10 +2,6 @@ Package: src:curl Pin: release n=bookworm-backports Pin-Priority: 600 -Package: src:e2fsprogs -Pin: release n=bookworm-backports -Pin-Priority: 600 - Package: src:elfutils Pin: release n=bookworm-backports Pin-Priority: 600 diff --git a/build-scripts/image-base.sh b/build-scripts/image-base.sh index 91376c5..002d740 100755 --- a/build-scripts/image-base.sh +++ b/build-scripts/image-base.sh @@ -27,7 +27,7 @@ grab_site_packages() { PYTHON_SITE_PACKAGES=$(grab_site_packages "docker.io/python:${PYTHONTAG}") [ -n "${PYTHON_SITE_PACKAGES:?}" ] -img="docker.io/rockdrilla/postgresql:base-v4" +img="docker.io/rockdrilla/postgresql:base-v5" buildah bud \ -f ./Dockerfile.base \ diff --git a/build-scripts/image-deps.sh b/build-scripts/image-deps.sh index 409042b..cedbce3 100755 --- a/build-scripts/image-deps.sh +++ b/build-scripts/image-deps.sh @@ -8,8 +8,8 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}" BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}" set +a -img="docker.io/rockdrilla/postgresql:deps-v4" -base="docker.io/rockdrilla/postgresql:base-v4" +img="docker.io/rockdrilla/postgresql:deps-v5" +base="docker.io/rockdrilla/postgresql:base-v5" exec buildah bud \ -f ./Dockerfile.deps \ diff --git a/build-scripts/image.sh b/build-scripts/image.sh index 07fa4d1..b4e186d 100755 --- a/build-scripts/image.sh +++ b/build-scripts/image.sh @@ -12,7 +12,7 @@ POSTGRESQL_VERSION="${1:-16.7}" PG_MAJOR="${POSTGRESQL_VERSION%%.*}" img="docker.io/rockdrilla/postgresql:${POSTGRESQL_VERSION}" -deps="docker.io/rockdrilla/postgresql:deps-v4" +deps="docker.io/rockdrilla/postgresql:deps-v5" c=$(buildah from --pull=missing "${deps}") [ -n "${c:?}" ] diff --git a/ep.sh b/ep.sh index e1ddb8b..38d580d 100755 --- a/ep.sh +++ b/ep.sh @@ -47,11 +47,11 @@ chown -h postgres:postgres "${PGDATA}" set -e if [ "$c" = postgres ] ; then - exec dumb-init docker-entrypoint.sh "$@" + exec catatonit -- docker-entrypoint.sh "$@" fi if [ "$(id -u)" != 0 ] ; then - exec dumb-init "$@" + exec catatonit -- "$@" fi -exec dumb-run-as.sh postgres dumb-init "$@" +exec dumb-run-as.sh postgres catatonit -- "$@" diff --git a/requirements.txt b/requirements.txt index 4becd4b..e9864fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ ## psycopg[c,pool] -psycopg-c==3.2.6 -typing_extensions==4.13.1 +psycopg-c==3.2.9 +typing_extensions==4.13.2 psycopg-pool==3.2.6 -psycopg[c,pool]==3.2.6 +psycopg[c,pool]==3.2.9 ## patroni[etcd3,kubernetes] -click==8.1.8 +click==8.2.1 wcwidth==0.2.13 prettytable==3.16.0 psutil==7.0.0 six==1.17.0 python-dateutil==2.9.0.post0 PyYAML==6.0.2 -urllib3==2.3.0 +urllib3==2.4.0 ydiff==1.4.2 dnspython==2.7.0 python-etcd==0.4.5 @@ -24,5 +24,5 @@ patroni[etcd3,kubernetes]==4.0.5 cdiff==1.0 pycparser==2.22 cffi==1.17.1 -cryptography==44.0.2 +cryptography==45.0.2 netaddr==1.3.0 diff --git a/scripts/dpkg-search.sh b/scripts/dpkg-search.sh new file mode 100755 index 0000000..304eac5 --- /dev/null +++ b/scripts/dpkg-search.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -ef +: "${1:?}" + +if dpkg-query --search "$1" ; then + exit 0 +fi + +case "$1" in +*\** | *\?* ) + env printf '%s does not support globs: %q\n' "${0##*/}" "$1" >&2 + exit 1 +;; +esac + +while read -r f ; do + [ -n "$f" ] || continue + dpkg-query --search "$f" || continue + exit 0 +done </dev/null | grep -Fxv -e "$1") +EOF + +exit 1