krd/postgresql-citus-patroni-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: krd:804d6b2544776a398aad4ee4e5dd61e4ec5fd28f
Branches Tags
krd:main
..
compare: krd:dea7d7cd0e5b2b29bd0e632e33ca23e5c420f536
Branches Tags
krd:main

No commits in common. "804d6b2544776a398aad4ee4e5dd61e4ec5fd28f" and "dea7d7cd0e5b2b29bd0e632e33ca23e5c420f536" have entirely different histories.
804d6b2544 ... dea7d7cd0e

15 changed files with 286 additions and 295 deletions
Show Stats Expand all files Collapse all files

15
Dockerfile
View File

@ -41,12 +41,6 @@ RUN d="@$(date '+%s')" ; \
find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \ find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} + find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
WORKDIR /pycache
RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \
| sort -zV \
| tar -C /usr/local --null -T - -cf - \
| tar -xf -
## --- ## ---
FROM deps AS postgresql FROM deps AS postgresql
@ -128,8 +122,7 @@ RUN apt-install.sh \
"postgresql-${PG_MAJOR}-unit" \ "postgresql-${PG_MAJOR}-unit" \
"postgresql-${PG_MAJOR}-wal2json" \ "postgresql-${PG_MAJOR}-wal2json" \
; \ ; \
apt-clean.sh ; \ apt-clean.sh
jdupes -1LSpr /usr/
## --- ## ---
@ -141,8 +134,7 @@ COPY /apt/sources.citus /etc/apt/sources.list.d/citus.sources
RUN apt-install.sh \ RUN apt-install.sh \
"postgresql-${PG_MAJOR}-citus-12.1" \ "postgresql-${PG_MAJOR}-citus-12.1" \
"postgresql-${PG_MAJOR}-topn" \ "postgresql-${PG_MAJOR}-topn" \
; apt-clean.sh ; \ ; apt-clean.sh
jdupes -1LSpr /usr/
VOLUME [ "${PGHOME}" ] VOLUME [ "${PGHOME}" ]
@ -154,7 +146,8 @@ SHELL [ "/bin/sh", "-ec" ]
COPY /Dockerfile /usr/local/share/ COPY /Dockerfile /usr/local/share/
## RFC: Python cache ## RFC: Python cache
COPY --from=pycache /pycache/ /usr/local/ ## TODO: reduce load by selecting only __pycache__ directories in either way
COPY --from=pycache /usr/local/lib/ /usr/local/lib/
COPY /ep.sh /usr/local/sbin/ COPY /ep.sh /usr/local/sbin/
COPY /postgres-shim.sh /usr/local/sbin/ COPY /postgres-shim.sh /usr/local/sbin/

133
Dockerfile.base
View File

@ -1,7 +1,7 @@
ARG PYTHONTAG=3.11.10-slim-bookworm ARG PYTHONTAG=3.11.9-slim-bookworm
FROM docker.io/python:${PYTHONTAG} AS base-upstream FROM docker.io/python:${PYTHONTAG} AS base-upstream
FROM base-upstream AS base-intermediate FROM base-upstream AS base
SHELL [ "/bin/sh", "-ec" ] SHELL [ "/bin/sh", "-ec" ]
COPY /Dockerfile.base /usr/local/share/ COPY /Dockerfile.base /usr/local/share/
@ -25,7 +25,6 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
# PIP_INDEX_URL="http://127.0.0.1:8081/repository/proxy_pypi/simple/" \ # PIP_INDEX_URL="http://127.0.0.1:8081/repository/proxy_pypi/simple/" \
# PIP_TRUSTED_HOST="localhost" # PIP_TRUSTED_HOST="localhost"
COPY /apt/preferences.backports /etc/apt/preferences.d/backports
COPY /apt/sources.debian /etc/apt/sources.list.d/debian.sources COPY /apt/sources.debian /etc/apt/sources.list.d/debian.sources
## prevent services from auto-starting, part 1 ## prevent services from auto-starting, part 1
@ -46,39 +45,7 @@ RUN divert_true() { divert-rm.sh "$1" ; ln -sv /bin/true "$1" ; } ; \
divert_true /usr/bin/deb-systemd-invoke divert_true /usr/bin/deb-systemd-invoke
RUN apt-env.sh apt-get update ; \ RUN apt-env.sh apt-get update ; \
apt-remove.sh \
ca-certificates \
e2fsprogs \
; \
apt-env.sh apt-get upgrade -y ; \ apt-env.sh apt-get upgrade -y ; \
apt-install.sh \
brotli \
cron \
curl \
dumb-init \
file \
gettext-base \
gnupg \
iproute2 \
iputils-ping \
jdupes \
jq \
less \
libnss-wrapper \
logrotate \
lsof \
ncurses-base \
netbase \
netcat-openbsd \
openssl \
procps \
psmisc \
tzdata \
vim \
xxd \
xz-utils \
zstd \
; \
apt-clean.sh apt-clean.sh
## perl-base: hardlink->symlink ## perl-base: hardlink->symlink
@ -107,7 +74,6 @@ RUN set -f ; \
chsh \ chsh \
cpgr \ cpgr \
cppw \ cppw \
crontab \
ctrlaltdel \ ctrlaltdel \
debugfs \ debugfs \
delpart \ delpart \
@ -240,106 +206,35 @@ RUN set -f ; \
done ; \ done ; \
done done
RUN apt-remove.sh \
ca-certificates \
e2fsprogs \
; \
apt-clean.sh
## "docker.io/python"-specific cleanup ## "docker.io/python"-specific cleanup
RUN rm -f /root/.wget-hsts RUN rm -f /root/.wget-hsts
RUN pip-env.sh pip list --format freeze \ RUN python-rm-cache.sh /usr/local ; \
pip-env.sh pip list --format freeze \
| grep -F '==' | awk -F= '{print $1}' \ | grep -F '==' | awk -F= '{print $1}' \
| xargs -r pip-env.sh pip install -U ; \ | xargs -r pip-env.sh pip install -U ; \
python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" python-rm-cache.sh "${PYTHON_SITE_PACKAGES}"
RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ RUN python-rm-cache.sh /usr/local ; \
libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
rm -rfv \ rm -rfv \
/usr/local/bin/idle* \ /usr/local/bin/idle* \
/usr/local/bin/pydoc* \
"${libpython}/ensurepip/_bundled" \ "${libpython}/ensurepip/_bundled" \
"${libpython}/idlelib" \ "${libpython}/idlelib" \
"${libpython}/pydoc.py" \
"${libpython}/pydoc_data" \
"${libpython}/tkinter" \ "${libpython}/tkinter" \
"${libpython}/turtle.py" \ "${libpython}/turtle.py" \
"${libpython}/turtledemo" \ "${libpython}/turtledemo" \
; \ ; :
rm -rfv \
"${PYTHON_SITE_PACKAGES}/pkg_resources/tests" \
"${PYTHON_SITE_PACKAGES}/setuptools/tests" \
"${PYTHON_SITE_PACKAGES}/setuptools/_distutils/tests" \
"${PYTHON_SITE_PACKAGES}/setuptools/_vendor/importlib_resources/tests" \
; \
find "${PYTHON_SITE_PACKAGES}/" -iname '*.exe' -ls -delete ; \
python-rm-cache.sh /usr/local
## adjust pip/certifi
RUN certifi_pem="${PYTHON_SITE_PACKAGES}/pip/_vendor/certifi/cacert.pem" ; \
rm -f "${certifi_pem}" ; \
ln -s /etc/ssl/certs/ca-certificates.crt "${certifi_pem}"
RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ RUN find /usr/local/sbin/ ! -type d -ls -delete ; \
find /run/ -mindepth 1 -ls -delete || : ; \ find /run/ -mindepth 1 -ls -delete || : ; \
install -d -m 01777 /run/lock ; \ install -d -m 01777 /run/lock
jdupes -1LSpr /usr/
## ---
FROM base-intermediate AS certs
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
## "2024.08.30"
ENV CERTIFI_COMMIT=325c2fde4f8eec10d682b09f3b0414dc05e69a81
# 'https://raw.githubusercontent.com/certifi/python-certifi'
ARG CERTIFI_BASE_URI='https://github.com/certifi/python-certifi/raw'
ARG CERTIFI_URI="${CERTIFI_BASE_URI}/${CERTIFI_COMMIT}/certifi/cacert.pem"
ADD "${CERTIFI_URI}" /tmp/certifi.crt
RUN apt-install.sh ca-certificates ; \
apt-clean.sh ; \
ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
ls -l "${ca_file}" ; \
## process certifi
certifi-extras.sh /tmp/certifi.crt ; \
openssl-cert-auto-pem.sh "${ca_file}" "${ca_file}.new" "${ca_file}.fp" ; \
mv -f "${ca_file}.new" "${ca_file}" ; \
chmod 0444 "${ca_file}" "${ca_file}.fp" ; \
ls -l "${ca_file}" "${ca_file}.fp"
## ---
FROM base-intermediate AS apt-gpg
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
ADD https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc /tmp/pgdg.gpg.bin
ADD https://packagecloud.io/citusdata/community/gpgkey /tmp/citus.gpg.bin
## process GPG keyrings
RUN pkg='gnupg' ; \
apt-install.sh ${pkg} ; \
gpg-export.sh /tmp/pgdg.gpg.bin /etc/apt/keyrings/pgdg.gpg.asc ; \
gpg-export.sh /tmp/citus.gpg.bin /etc/apt/keyrings/citus.gpg.asc ; \
apt-remove.sh ${pkg}
COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources
COPY /apt/sources.citus /etc/apt/sources.list.d/citus.sources
## verify sources!
RUN apt-env.sh apt-get update ; \
apt-clean.sh
## ---
FROM base-intermediate AS base
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/
ENTRYPOINT [ ] ENTRYPOINT [ ]
CMD [ "bash" ] CMD [ "bash" ]

113
Dockerfile.deps
View File

@ -1,5 +1,52 @@
ARG BASE_IMAGE FROM docker.io/rockdrilla/postgresql:base-v1 AS base
FROM ${BASE_IMAGE} AS base
## ---
FROM base AS certs
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
## consult https://github.com/certifi/python-certifi/
ENV CERTIFI_COMMIT=bd8153872e9c6fc98f4023df9c2deaffea2fa463
RUN apt-install.sh ca-certificates curl ; \
apt-clean.sh ; \
## process certifi
ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
ls -l "${ca_file}" ; \
certifi-extras.sh ; \
openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp" ; \
chmod 0444 "${ca_file}" "${ca_file}.fp" ; \
ls -l "${ca_file}" "${ca_file}.fp"
## ---
FROM base AS apt-gpg
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
ADD https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc /tmp/pgdg.gpg.bin
ADD https://packagecloud.io/citusdata/community/gpgkey /tmp/citus.gpg.bin
## process GPG keyrings
RUN pkg='gnupg' ; \
apt-install.sh ${pkg} ; \
gpg-export.sh /tmp/pgdg.gpg.bin /etc/apt/keyrings/pgdg.gpg.asc ; \
gpg-export.sh /tmp/citus.gpg.bin /etc/apt/keyrings/citus.gpg.asc ; \
apt-remove.sh ${pkg}
COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources
COPY /apt/sources.citus /etc/apt/sources.list.d/citus.sources
## verify sources!
RUN apt-env.sh apt-get update ; \
apt-clean.sh
## --- ## ---
@ -8,17 +55,21 @@ SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/ COPY /scripts/* /usr/local/sbin/
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/
COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources
COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg
COPY /requirements.txt /tmp/
ENV DEV_PACKAGES='libffi-dev libpq-dev libyaml-dev' ENV DEV_PACKAGES='libffi-dev libpq-dev libyaml-dev'
# psutil # psutil
ENV CIBUILDWHEEL=1 ENV CIBUILDWHEEL=1
# pyyaml # pyyaml
ENV PYYAML_FORCE_CYTHON=1 ENV PYYAML_FORCE_CYTHON=1
COPY /requirements.txt /tmp/
RUN w=$(mktemp -d) ; : "${w:?}" ; \ RUN w=$(mktemp -d) ; : "${w:?}" ; \
{ apt-mark showauto ; apt-mark showmanual ; } | sort -uV > "$w/t0" ; \ { apt-mark showauto ; apt-mark showmanual ; } | sort -uV > "$w/t0" ; \
printf '%s\n' ${DEV_PACKAGES} | sort -uV > "$w/t1" ; \ printf '%s\n' ${DEV_PACKAGES} | sort -uV > "$w/t1" ; \
@ -37,44 +88,34 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \
-r /tmp/requirements.txt \ -r /tmp/requirements.txt \
; \ ; \
pip-env.sh pip uninstall -y 'cython' ; \ pip-env.sh pip uninstall -y 'cython' ; \
python-rm-cache.sh /usr/local ; \ python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" ; \
rm -rf \ rm -rf \
"${PYTHON_SITE_PACKAGES}/etcd/tests" \ "${PYTHON_SITE_PACKAGES}/etcd/tests" \
"${PYTHON_SITE_PACKAGES}/netaddr/tests" \
"${PYTHON_SITE_PACKAGES}/psutil/tests" \ "${PYTHON_SITE_PACKAGES}/psutil/tests" \
; \ ; \
truncate -s 0 \
"${PYTHON_SITE_PACKAGES}/netaddr/eui/iab.idx" \
"${PYTHON_SITE_PACKAGES}/netaddr/eui/iab.txt" \
"${PYTHON_SITE_PACKAGES}/netaddr/eui/oui.txt" \
"${PYTHON_SITE_PACKAGES}/netaddr/eui/oui.idx" \
; \
find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \
echo ; \ echo ; \
find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -printf '%p\0' \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -printf '%p\0' \
| sed -zE '/rust/d' \ | sed -zE '/rust/d' \
| xargs -0r strip --verbose --strip-debug --strip-unneeded ; \ | xargs -0r strip --verbose --strip-debug ; \
echo ; \ echo ; \
find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \
apt-remove.sh build-essential ; \ apt-remove.sh build-essential ; \
apt-clean.sh apt-clean.sh
## avoid changing already present packages
RUN find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
| sed -zEn \
-e '/^((pip|setuptools|wheel)-.+\.dist-info|distutils-precedence\.pth|_distutils_hack|pip|pkg_resources|setuptools|wheel)$/p' \
| env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \
rm -rf
## --- ## ---
FROM base AS deps FROM base
SHELL [ "/bin/sh", "-ec" ] SHELL [ "/bin/sh", "-ec" ]
COPY /Dockerfile.deps /usr/local/share/ COPY /Dockerfile.deps /usr/local/share/
COPY /scripts/* /usr/local/sbin/ COPY /scripts/* /usr/local/sbin/
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/
COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources
COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg
@ -87,6 +128,33 @@ RUN f="${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \
xargs -a "$f" apt-install.sh ; \ xargs -a "$f" apt-install.sh ; \
apt-clean.sh apt-clean.sh
RUN apt-install.sh \
brotli \
curl \
dumb-init \
file \
gettext-base \
gnupg \
iproute2 \
iputils-ping \
jq \
less \
libnss-wrapper \
lsof \
ncurses-base \
netbase \
netcat-openbsd \
openssl \
procps \
psmisc \
tzdata \
vim \
xxd \
xz-utils \
zstd \
; \
apt-clean.sh
## set up locales! ## set up locales!
RUN _lang=en_US.UTF8 ; \ RUN _lang=en_US.UTF8 ; \
{ \ { \
@ -106,5 +174,4 @@ RUN _lang=en_US.UTF8 ; \
locale -a | grep -Fixq "${_lang}" locale -a | grep -Fixq "${_lang}"
ENV LANG=en_US.UTF8 ENV LANG=en_US.UTF8
RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ RUN find /usr/local/sbin/ ! -type d -ls -delete
jdupes -1LSpr /usr/

4
apt/preferences.backports
View File

@ -1,4 +0,0 @@
## example:
# Package: src:curl
# Pin: release n=bookworm-backports
# Pin-Priority: 600

6
apt/sources.debian
View File

@ -1,11 +1,11 @@
Types: deb Types: deb
URIs: http://deb.debian.org/debian URIs: http://deb.debian.org/debian
Suites: bookworm bookworm-updates bookworm-proposed-updates bookworm-backports Suites: bookworm bookworm-updates bookworm-proposed-updates
Components: main Components: main contrib non-free
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
Types: deb Types: deb
URIs: http://deb.debian.org/debian-security URIs: http://deb.debian.org/debian-security
Suites: bookworm-security Suites: bookworm-security
Components: main Components: main contrib non-free
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg

4
build-scripts/image-base.sh
View File

@ -8,7 +8,7 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}" BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
set +a set +a
PYTHONTAG="${PYTHONTAG:-3.11.10-slim-bookworm}" PYTHONTAG="${PYTHONTAG:-3.11.9-slim-bookworm}"
grab_site_packages() { grab_site_packages() {
podman run \ podman run \
@ -27,7 +27,7 @@ grab_site_packages() {
PYTHON_SITE_PACKAGES=$(grab_site_packages "docker.io/python:${PYTHONTAG}") PYTHON_SITE_PACKAGES=$(grab_site_packages "docker.io/python:${PYTHONTAG}")
[ -n "${PYTHON_SITE_PACKAGES:?}" ] [ -n "${PYTHON_SITE_PACKAGES:?}" ]
img="docker.io/rockdrilla/postgresql:base-v2" img="docker.io/rockdrilla/postgresql:base-v1"
buildah bud \ buildah bud \
-f ./Dockerfile.base \ -f ./Dockerfile.base \

6
build-scripts/image-deps.sh
View File

@ -8,11 +8,9 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}" BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
set +a set +a
img="docker.io/rockdrilla/postgresql:deps-v2" img="docker.io/rockdrilla/postgresql:deps-v1"
base="docker.io/rockdrilla/postgresql:base-v2"
exec buildah bud \ exec buildah bud \
-f ./Dockerfile.deps \ -f ./Dockerfile.deps \
-t "${img}" \ -t "${img}" \
--pull=missing --no-cache \ --pull=missing --no-cache
--build-arg "BASE_IMAGE=${base}" \

4
build-scripts/image.sh
View File

@ -8,11 +8,11 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}"
BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}" BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}"
set +a set +a
POSTGRESQL_VERSION="${1:-16.6}" POSTGRESQL_VERSION="${1:-16.4}"
PG_MAJOR="${POSTGRESQL_VERSION%%.*}" PG_MAJOR="${POSTGRESQL_VERSION%%.*}"
img="docker.io/rockdrilla/postgresql:${POSTGRESQL_VERSION}" img="docker.io/rockdrilla/postgresql:${POSTGRESQL_VERSION}"
deps="docker.io/rockdrilla/postgresql:deps-v2" deps="docker.io/rockdrilla/postgresql:deps-v1"
c=$(buildah from --pull=missing "${deps}") c=$(buildah from --pull=missing "${deps}")
[ -n "${c:?}" ] [ -n "${c:?}" ]

70
extra-scripts/certifi-extras.sh
View File

@ -1,41 +1,89 @@
#!/bin/sh #!/bin/sh
set -ef set -ef
certifi_uri="https://raw.githubusercontent.com/certifi/python-certifi/${CERTIFI_COMMIT:?}/certifi/cacert.pem"
dst_dir=/usr/local/share/ca-certificates dst_dir=/usr/local/share/ca-certificates
w=$(mktemp -d) ; : "${w:?}" w=$(mktemp -d) ; : "${w:?}"
w_cleanup() { w_cleanup() {
[ -z "$w" ] || ls -lA "$w/" >&2 [ -z "$w" ] || ls -lA "$w/"
[ -z "$w" ] || rm -rf "$w" [ -z "$w" ] || rm -rf "$w"
unset w unset w
exit "${1:-0}" exit "${1:-0}"
} }
curl -sSL "${certifi_uri}" > "$w/certifi.crt"
def_bundle='/etc/ssl/certs/ca-certificates.crt' def_bundle='/etc/ssl/certs/ca-certificates.crt'
openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp" openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem"
openssl-cert-auto-pem.sh "$w/certifi.crt" > "$w/certifi.pem"
[ -s "$w/cacert.pem" ] || w_cleanup 1 [ -s "$w/cacert.pem" ] || w_cleanup 1
[ -s "$w/cacert.fp" ] || w_cleanup 1
openssl-cert-auto-pem.sh "$1" "$w/certifi.pem" "$w/certifi.fp" "$w/certifi.off"
[ -s "$w/certifi.pem" ] || w_cleanup 1 [ -s "$w/certifi.pem" ] || w_cleanup 1
[ -s "$w/certifi.fp" ] || w_cleanup 1
bundle_offsets() {
awk '
BEGIN {
OFS = ","
m_begin="-----BEGIN CERTIFICATE-----"
m_end="-----END CERTIFICATE-----"
i_begin = 0
}
$0 == m_begin { i_begin = NR ; }
$0 == m_end {
if (i_begin > 0) {
print i_begin,NR
i_begin = 0
}
}
' "$1"
}
bundle_offsets "$w/cacert.pem" > "$w/cacert.off"
bundle_offsets "$w/certifi.pem" > "$w/certifi.off"
[ -s "$w/cacert.off" ] || w_cleanup 1
[ -s "$w/certifi.off" ] || w_cleanup 1 [ -s "$w/certifi.off" ] || w_cleanup 1
bundle_fingerprints() {
local a
while read -r a ; do
[ -n "$a" ] || continue
{
sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint -sha256 \
|| \
sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint
} | tr '[:upper:]' '[:lower:]'
done < "$2"
}
bundle_fingerprints "$w/cacert.pem" "$w/cacert.off" | sort -uV > "$w/cacert.fp"
bundle_fingerprints "$w/certifi.pem" "$w/certifi.off" | sort -uV > "$w/certifi.fp"
[ -s "$w/cacert.fp" ] || w_cleanup 1
[ -s "$w/certifi.fp" ] || w_cleanup 1
set +e set +e
grep -Fxnv -f "$w/cacert.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/diff.ln" grep -Fxv -f "$w/cacert.fp" "$w/certifi.fp" > "$w/diff.fp"
set -e set -e
if [ -s "$w/diff.ln" ] ; then if [ -s "$w/diff.fp" ] ; then
terse_fingerprint() { cut -d = -f 2- | tr -cd '[:alnum:]' ; } set +e
grep -Fxn -f "$w/diff.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/records.diff"
set -e
terse_fingerprint() {
cut -d = -f 2- | tr -cd '[:alnum:]'
}
mkdir "$w/extras"
while read -r n ; do while read -r n ; do
[ -n "$n" ] || continue [ -n "$n" ] || continue
fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint) fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint)
off=$(sed -ne "${n}p" "$w/certifi.off") off=$(sed -ne "${n}p" "$w/certifi.off")
sed -ne "${off}p" "$w/certifi.pem" > "${dst_dir}/certifi-${fp}.crt" sed -ne "${off}p" "$w/certifi.pem" | openssl x509 > "${dst_dir}/certifi-${fp}.crt"
done < "$w/diff.ln" done < "$w/records.diff"
fi fi
rm -rf "$w" ; unset w rm -rf "$w" ; unset w

23
requirements.txt
View File

@ -1,28 +1,27 @@
## psycopg[c,pool] ## psycopg[c,pool]
psycopg-c==3.2.3 psycopg-c==3.2.1
typing_extensions==4.12.2 typing_extensions==4.12.2
psycopg-pool==3.2.4 psycopg-pool==3.2.2
psycopg[c,pool]==3.2.3 psycopg[c,pool]==3.2.1
## patroni[etcd3,kubernetes] ## patroni[etcd3,kubernetes]
click==8.1.7 click==8.1.7
wcwidth==0.2.13 wcwidth==0.2.13
prettytable==3.12.0 prettytable==3.11.0
psutil==6.1.0 psutil==6.0.0
six==1.16.0 six==1.16.0
python-dateutil==2.9.0.post0 python-dateutil==2.9.0.post0
PyYAML==6.0.2 PyYAML==6.0.2
urllib3==2.2.3 urllib3==2.2.2
ydiff==1.4.2 ydiff==1.3
dnspython==2.7.0 dnspython==2.6.1
python-etcd==0.4.5 python-etcd==0.4.5
patroni[etcd3,kubernetes]==4.0.4 patroni[etcd3,kubernetes]==3.3.2
## misc ## misc
cdiff==1.0 cdiff==1.0
pycparser==2.22 pycparser==2.22
cffi==1.17.1 cffi==1.17.0
cryptography==44.0.0 cryptography==43.0.0
netaddr==1.3.0

12
scripts/apt-clean.sh
View File

@ -23,15 +23,11 @@ find /var/cache/debconf/ ! -type d -wholename '/var/cache/debconf/*-old' -delete
__t=$(mktemp) ; : "${__t:?}" __t=$(mktemp) ; : "${__t:?}"
debconf_trim_i18n() { debconf_trim_i18n() {
mawk 'BEGIN { m = 0; } mawk 'BEGIN { m = 0 }
$0 == "" { print; } $0 == "" { print }
/^[^[:space:]]/ { /^[^[:space:]]/ {
if ($1 ~ "\.[Uu][Tt][Ff]-?8:") { if ($1 ~ "\.[Uu][Tt][Ff]-?8:") { m = 1; next; }
m = 1; m = 0; print $0;
next;
}
m = 0;
print $0;
} }
/^[[:space:]]/ { /^[[:space:]]/ {
if (m == 1) next; if (m == 1) next;

5
scripts/apt-install.sh
View File

@ -22,9 +22,8 @@ _apt_update() {
} }
_dpkg_avail_hack() { _dpkg_avail_hack() {
: "${DPKG_ADMINDIR:=/var/lib/dpkg}"
VERSION_CODENAME=$(. /etc/os-release ; printf '%s' "${VERSION_CODENAME}") || : VERSION_CODENAME=$(. /etc/os-release ; printf '%s' "${VERSION_CODENAME}") || :
f="${DPKG_ADMINDIR}/available" f="${DPKG_ADMINDIR:-/var/lib/dpkg}/available"
# if ${VERSION_CODENAME} is empty then we're on Debian sid or so :) # if ${VERSION_CODENAME} is empty then we're on Debian sid or so :)
case "${VERSION_CODENAME}" in case "${VERSION_CODENAME}" in
stretch | buster | bionic | focal ) stretch | buster | bionic | focal )
@ -32,7 +31,7 @@ _dpkg_avail_hack() {
if [ -s "$f" ] ; then if [ -s "$f" ] ; then
return return
fi fi
/usr/lib/dpkg/methods/apt/update "${DPKG_ADMINDIR}" apt apt /usr/lib/dpkg/methods/apt/update "${DPKG_ADMINDIR:-/var/lib/dpkg}" apt apt
;; ;;
* ) * )
touch "$f" touch "$f"

92
scripts/openssl-cert-auto-pem.sh
View File

@ -13,88 +13,38 @@ me=${0##*/}
w=$(mktemp -d) || exit 1 w=$(mktemp -d) || exit 1
w_cleanup() { w_cleanup() {
[ -z "$w" ] || ls -lA "$w/" >&2 [ -z "$w" ] || ls -lA "$w/"
[ -z "$w" ] || rm -rf "$w" [ -z "$w" ] || rm -rf "$w"
unset w unset w
exit "${1:-0}" exit "${1:-0}"
} }
bundle_offsets() {
mawk 'BEGIN { OFS = ","; i_begin = 0; }
$0 == "-----BEGIN CERTIFICATE-----" {
i_begin = NR;
}
$0 == "-----END CERTIFICATE-----" {
if (i_begin > 0) {
print i_begin, NR;
i_begin = 0;
}
}' "$1"
}
bundle_fingerprints() {
local x f
while read -r x ; do
[ -n "$x" ] || continue
f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint -sha256)
[ -n "$f" ] || f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint)
[ -n "$f" ] || continue
printf '%s\n' "$f" | tr '[:upper:]' '[:lower:]'
done < "$2"
}
openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1 openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1
[ -s "$w/cert.pem" ] || w_cleanup 1 [ -s "$w/cert.pem" ] || w_cleanup 1
tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt" tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt"
[ -s "$w/cert.txt" ] || w_cleanup 1 [ -s "$w/cert.txt" ] || w_cleanup 1
rm -f "$w/cert.pem"
bundle_offsets "$w/cert.txt" > "$w/cert.off" awk '
[ -s "$w/cert.off" ] || w_cleanup 1 BEGIN {
OFS = ","
m_begin="-----BEGIN CERTIFICATE-----"
m_end="-----END CERTIFICATE-----"
i_begin = 0
}
$0 == m_begin { i_begin = NR ; }
$0 == m_end {
if (i_begin > 0) {
print i_begin,NR
i_begin = 0
}
}
' "$w/cert.txt" > "$w/cert.offsets"
[ -s "$w/cert.offsets" ] || w_cleanup 1
bundle_fingerprints "$w/cert.txt" "$w/cert.off" > "$w/cert.fp.all" while read -r a ; do
[ -s "$w/cert.fp.all" ] || w_cleanup 1 [ -n "$a" ] || continue
sort -uV < "$w/cert.fp.all" > "$w/cert.fp" sed -ne "${a}p" "$w/cert.txt"
while read -r fp ; do done < "$w/cert.offsets"
[ -n "${fp}" ] || continue
n=$(grep -m1 -Fxn -e "${fp}" "$w/cert.fp.all" | cut -d : -f 1)
[ -n "$n" ] || continue
off=$(sed -ne "${n}p" "$w/cert.off")
[ -n "${off}" ] || continue
sed -ne "${off}p" "$w/cert.txt"
done < "$w/cert.fp" > "$w/cert.pem"
[ -s "$w/cert.pem" ] || w_cleanup 1
rm -f "$w/cert.txt" "$w/cert.off" "$w/cert.fp.all"
if [ -n "$2" ] ; then
while : ; do
if [ -e "$2" ] ; then
[ -f "$2" ] || break
fi
cat > "$2"
break ; done
else
cat
fi < "$w/cert.pem"
while [ -n "$3" ] ; do
if [ -e "$3" ] ; then
[ -f "$3" ] || break
fi
cat "$w/cert.fp" > "$3"
break ; done
while [ -n "$4" ] ; do
if [ -e "$4" ] ; then
[ -f "$4" ] || break
fi
bundle_offsets "$w/cert.pem" > "$4"
break ; done
rm -rf "$w" ; unset w rm -rf "$w" ; unset w

52
scripts/openssl-cert-fingerprint.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/sh
set -f
[ $# -gt 0 ] || exit 0
me=${0##*/}
[ -n "$1" ] || exit 1
[ -f "$1" ] || {
env printf '%s: not a file or does not exist: %q\n' "${me}" "$1" >&2
exit 1
}
[ -s "$1" ] || exit 0
w=$(mktemp -d) || exit 1
w_cleanup() {
[ -z "$w" ] || ls -lA "$w/"
[ -z "$w" ] || rm -rf "$w"
unset w
exit "${1:-0}"
}
openssl-cert-auto-pem.sh "$1" > "$w/cert.pem" || w_cleanup 1
[ -s "$w/cert.pem" ] || w_cleanup 1
awk '
BEGIN {
OFS = ","
m_begin="-----BEGIN CERTIFICATE-----"
m_end="-----END CERTIFICATE-----"
i_begin = 0
}
$0 == m_begin { i_begin = NR ; }
$0 == m_end {
if (i_begin > 0) {
print i_begin,NR
i_begin = 0
}
}
' "$w/cert.pem" > "$w/cert.off"
[ -s "$w/cert.off" ] || w_cleanup 1
while read -r a ; do
[ -n "$a" ] || continue
{
sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint -sha256 \
|| \
sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint
} | tr '[:upper:]' '[:lower:]'
done < "$w/cert.off"
w_cleanup 0

2
scripts/python-rm-cache.sh
View File

@ -1,8 +1,6 @@
#!/bin/sh #!/bin/sh
set -f set -f
for i ; do for i ; do
[ -n "$i" ] || continue
[ -d "$i" ] || continue
find "$i/" -name __pycache__ -exec rm -rf {} + find "$i/" -name __pycache__ -exec rm -rf {} +
find "$i/" ! -type d -name '*.py[co]' -exec rm -f {} + find "$i/" ! -type d -name '*.py[co]' -exec rm -f {} +
done done