diff --git a/Dockerfile b/Dockerfile index a1259f5..7c151dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,12 @@ RUN d="@$(date '+%s')" ; \ find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \ find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} + +WORKDIR /pycache +RUN find /usr/local/ -type f -name '*.py[co]' -printf '%P\0' \ + | sort -zV \ + | tar -C /usr/local --null -T - -cf - \ + | tar -xf - + ## --- FROM deps AS postgresql @@ -66,7 +72,7 @@ RUN sed "s/%{PG_MAJOR}/${PG_MAJOR}/g" < /tmp/pgdg-ver.sources > "/etc/apt/source sed "s/%{PG_MAJOR}/${PG_MAJOR}/g" < /tmp/pgdg-ver.prefs > "/etc/apt/preferences.d/pgdg-${PG_MAJOR}" ; \ rm -f /tmp/pgdg-ver.sources /tmp/pgdg-ver.prefs -COPY --from=postgresql-upstream /etc/postgresql-common/createcluster.conf /etc/postgresql-common/ +COPY --from=postgresql-upstream /etc/postgresql-common/createcluster.conf /etc/postgresql-common/ RUN apt-install.sh postgresql-common ; \ apt-install.sh \ "postgresql-${PG_MAJOR}" \ @@ -75,12 +81,12 @@ RUN apt-install.sh postgresql-common ; \ f="/usr/share/postgresql/${PG_MAJOR}/postgresql.conf.sample" ; \ dpkg-divert --add --rename --divert "$f.dpkg" "$f" ; \ ln -sv ../postgresql.conf.sample "/usr/share/postgresql/${PG_MAJOR}/" -COPY --from=postgresql-upstream /usr/share/postgresql/postgresql.conf.sample /usr/share/postgresql/ +COPY --from=postgresql-upstream /usr/share/postgresql/postgresql.conf.sample /usr/share/postgresql/ ENV PATH=${PATH}:/usr/lib/postgresql/${PG_MAJOR}/bin -COPY --from=postgresql-upstream /docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ -COPY --from=postgresql-upstream /usr/local/bin/*.sh /usr/local/bin/ +COPY --from=postgresql-upstream /docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ +COPY --from=postgresql-upstream /usr/local/bin/*.sh /usr/local/bin/ ## compatibility ;) RUN ln -sv /usr/local/sbin/dumb-run-as.sh /usr/local/bin/gosu @@ -122,7 +128,8 @@ RUN apt-install.sh \ "postgresql-${PG_MAJOR}-unit" \ "postgresql-${PG_MAJOR}-wal2json" \ ; \ - apt-clean.sh + apt-clean.sh ; \ + jdupes -1LSpr /usr/ ## --- @@ -134,7 +141,8 @@ COPY /apt/sources.citus /etc/apt/sources.list.d/citus.sources RUN apt-install.sh \ "postgresql-${PG_MAJOR}-citus-12.1" \ "postgresql-${PG_MAJOR}-topn" \ - ; apt-clean.sh + ; apt-clean.sh ; \ + jdupes -1LSpr /usr/ VOLUME [ "${PGHOME}" ] @@ -146,8 +154,7 @@ SHELL [ "/bin/sh", "-ec" ] COPY /Dockerfile /usr/local/share/ ## RFC: Python cache -## TODO: reduce load by selecting only __pycache__ directories in either way -COPY --from=pycache /usr/local/lib/ /usr/local/lib/ +COPY --from=pycache /pycache/ /usr/local/ COPY /ep.sh /usr/local/sbin/ COPY /postgres-shim.sh /usr/local/sbin/ diff --git a/Dockerfile.base b/Dockerfile.base index e163bb7..2f940ed 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,7 +1,7 @@ ARG PYTHONTAG=3.11.9-slim-bookworm FROM docker.io/python:${PYTHONTAG} AS base-upstream -FROM base-upstream AS base +FROM base-upstream AS base-intermediate SHELL [ "/bin/sh", "-ec" ] COPY /Dockerfile.base /usr/local/share/ @@ -25,7 +25,8 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \ # PIP_INDEX_URL="http://127.0.0.1:8081/repository/proxy_pypi/simple/" \ # PIP_TRUSTED_HOST="localhost" -COPY /apt/sources.debian /etc/apt/sources.list.d/debian.sources +COPY /apt/preferences.backports /etc/apt/preferences.d/backports +COPY /apt/sources.debian /etc/apt/sources.list.d/debian.sources ## prevent services from auto-starting, part 1 RUN s='/usr/sbin/policy-rc.d' ; b='/usr/bin/policy-rc.d' ; \ @@ -45,7 +46,39 @@ RUN divert_true() { divert-rm.sh "$1" ; ln -sv /bin/true "$1" ; } ; \ divert_true /usr/bin/deb-systemd-invoke RUN apt-env.sh apt-get update ; \ + apt-remove.sh \ + ca-certificates \ + e2fsprogs \ + ; \ apt-env.sh apt-get upgrade -y ; \ + apt-install.sh \ + brotli \ + cron \ + curl \ + dumb-init \ + file \ + gettext-base \ + gnupg \ + iproute2 \ + iputils-ping \ + jdupes \ + jq \ + less \ + libnss-wrapper \ + logrotate \ + lsof \ + ncurses-base \ + netbase \ + netcat-openbsd \ + openssl \ + procps \ + psmisc \ + tzdata \ + vim \ + xxd \ + xz-utils \ + zstd \ + ; \ apt-clean.sh ## perl-base: hardlink->symlink @@ -74,6 +107,7 @@ RUN set -f ; \ chsh \ cpgr \ cppw \ + crontab \ ctrlaltdel \ debugfs \ delpart \ @@ -206,35 +240,106 @@ RUN set -f ; \ done ; \ done -RUN apt-remove.sh \ - ca-certificates \ - e2fsprogs \ - ; \ - apt-clean.sh - ## "docker.io/python"-specific cleanup RUN rm -f /root/.wget-hsts -RUN python-rm-cache.sh /usr/local ; \ - pip-env.sh pip list --format freeze \ +RUN pip-env.sh pip list --format freeze \ | grep -F '==' | awk -F= '{print $1}' \ | xargs -r pip-env.sh pip install -U ; \ python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" -RUN python-rm-cache.sh /usr/local ; \ - libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ +RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ rm -rfv \ /usr/local/bin/idle* \ + /usr/local/bin/pydoc* \ "${libpython}/ensurepip/_bundled" \ "${libpython}/idlelib" \ + "${libpython}/pydoc.py" \ + "${libpython}/pydoc_data" \ "${libpython}/tkinter" \ "${libpython}/turtle.py" \ "${libpython}/turtledemo" \ - ; : + ; \ + rm -rfv \ + "${PYTHON_SITE_PACKAGES}/pkg_resources/tests" \ + "${PYTHON_SITE_PACKAGES}/setuptools/tests" \ + "${PYTHON_SITE_PACKAGES}/setuptools/_distutils/tests" \ + "${PYTHON_SITE_PACKAGES}/setuptools/_vendor/importlib_resources/tests" \ + ; \ + find "${PYTHON_SITE_PACKAGES}/" -iname '*.exe' -ls -delete ; \ + python-rm-cache.sh /usr/local + +## adjust pip/certifi +RUN certifi_pem="${PYTHON_SITE_PACKAGES}/pip/_vendor/certifi/cacert.pem" ; \ + rm -f "${certifi_pem}" ; \ + ln -s /etc/ssl/certs/ca-certificates.crt "${certifi_pem}" RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ find /run/ -mindepth 1 -ls -delete || : ; \ - install -d -m 01777 /run/lock + install -d -m 01777 /run/lock ; \ + jdupes -1LSpr /usr/ + +## --- + +FROM base-intermediate AS certs +SHELL [ "/bin/sh", "-ec" ] + +COPY /scripts/* /usr/local/sbin/ +COPY /extra-scripts/* /usr/local/sbin/ + +## "2024.08.30" +ENV CERTIFI_COMMIT=325c2fde4f8eec10d682b09f3b0414dc05e69a81 + +# 'https://raw.githubusercontent.com/certifi/python-certifi' +ARG CERTIFI_BASE_URI='https://github.com/certifi/python-certifi/raw' + +ARG CERTIFI_URI="${CERTIFI_BASE_URI}/${CERTIFI_COMMIT}/certifi/cacert.pem" +ADD "${CERTIFI_URI}" /tmp/certifi.crt + +RUN apt-install.sh ca-certificates ; \ + apt-clean.sh ; \ + ca_file='/etc/ssl/certs/ca-certificates.crt' ; \ + ls -l "${ca_file}" ; \ + ## process certifi + certifi-extras.sh /tmp/certifi.crt ; \ + openssl-cert-auto-pem.sh "${ca_file}" "${ca_file}.new" "${ca_file}.fp" ; \ + mv -f "${ca_file}.new" "${ca_file}" ; \ + chmod 0444 "${ca_file}" "${ca_file}.fp" ; \ + ls -l "${ca_file}" "${ca_file}.fp" + +## --- + +FROM base-intermediate AS apt-gpg +SHELL [ "/bin/sh", "-ec" ] + +COPY /scripts/* /usr/local/sbin/ +COPY /extra-scripts/* /usr/local/sbin/ + +COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ + +ADD https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc /tmp/pgdg.gpg.bin +ADD https://packagecloud.io/citusdata/community/gpgkey /tmp/citus.gpg.bin + +## process GPG keyrings +RUN pkg='gnupg' ; \ + apt-install.sh ${pkg} ; \ + gpg-export.sh /tmp/pgdg.gpg.bin /etc/apt/keyrings/pgdg.gpg.asc ; \ + gpg-export.sh /tmp/citus.gpg.bin /etc/apt/keyrings/citus.gpg.asc ; \ + apt-remove.sh ${pkg} + +COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources +COPY /apt/sources.citus /etc/apt/sources.list.d/citus.sources + +## verify sources! +RUN apt-env.sh apt-get update ; \ + apt-clean.sh + +## --- + +FROM base-intermediate AS base + +COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ +COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/ ENTRYPOINT [ ] CMD [ "bash" ] diff --git a/Dockerfile.deps b/Dockerfile.deps index daeb6b3..4557809 100644 --- a/Dockerfile.deps +++ b/Dockerfile.deps @@ -1,52 +1,5 @@ -FROM docker.io/rockdrilla/postgresql:base-v1 AS base - -## --- - -FROM base AS certs -SHELL [ "/bin/sh", "-ec" ] - -COPY /scripts/* /usr/local/sbin/ -COPY /extra-scripts/* /usr/local/sbin/ - -## consult https://github.com/certifi/python-certifi/ -ENV CERTIFI_COMMIT=bd8153872e9c6fc98f4023df9c2deaffea2fa463 - -RUN apt-install.sh ca-certificates curl ; \ - apt-clean.sh ; \ - ## process certifi - ca_file='/etc/ssl/certs/ca-certificates.crt' ; \ - ls -l "${ca_file}" ; \ - certifi-extras.sh ; \ - openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp" ; \ - chmod 0444 "${ca_file}" "${ca_file}.fp" ; \ - ls -l "${ca_file}" "${ca_file}.fp" - -## --- - -FROM base AS apt-gpg -SHELL [ "/bin/sh", "-ec" ] - -COPY /scripts/* /usr/local/sbin/ -COPY /extra-scripts/* /usr/local/sbin/ - -COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ - -ADD https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc /tmp/pgdg.gpg.bin -ADD https://packagecloud.io/citusdata/community/gpgkey /tmp/citus.gpg.bin - -## process GPG keyrings -RUN pkg='gnupg' ; \ - apt-install.sh ${pkg} ; \ - gpg-export.sh /tmp/pgdg.gpg.bin /etc/apt/keyrings/pgdg.gpg.asc ; \ - gpg-export.sh /tmp/citus.gpg.bin /etc/apt/keyrings/citus.gpg.asc ; \ - apt-remove.sh ${pkg} - -COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources -COPY /apt/sources.citus /etc/apt/sources.list.d/citus.sources - -## verify sources! -RUN apt-env.sh apt-get update ; \ - apt-clean.sh +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS base ## --- @@ -55,21 +8,17 @@ SHELL [ "/bin/sh", "-ec" ] COPY /scripts/* /usr/local/sbin/ -COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ - -COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/ - COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg +COPY /requirements.txt /tmp/ + ENV DEV_PACKAGES='libffi-dev libpq-dev libyaml-dev' # psutil ENV CIBUILDWHEEL=1 # pyyaml ENV PYYAML_FORCE_CYTHON=1 -COPY /requirements.txt /tmp/ - RUN w=$(mktemp -d) ; : "${w:?}" ; \ { apt-mark showauto ; apt-mark showmanual ; } | sort -uV > "$w/t0" ; \ printf '%s\n' ${DEV_PACKAGES} | sort -uV > "$w/t1" ; \ @@ -88,7 +37,7 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \ -r /tmp/requirements.txt \ ; \ pip-env.sh pip uninstall -y 'cython' ; \ - python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" ; \ + python-rm-cache.sh /usr/local ; \ rm -rf \ "${PYTHON_SITE_PACKAGES}/etcd/tests" \ "${PYTHON_SITE_PACKAGES}/psutil/tests" \ @@ -97,30 +46,33 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \ echo ; \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -printf '%p\0' \ | sed -zE '/rust/d' \ - | xargs -0r strip --verbose --strip-debug ; \ + | xargs -0r strip --verbose --strip-debug --strip-unneeded ; \ echo ; \ find "${PYTHON_SITE_PACKAGES}/" -type f -name '*.so*' -exec ls -l {} + ; \ apt-remove.sh build-essential ; \ apt-clean.sh +## avoid changing already present packages +RUN find "${PYTHON_SITE_PACKAGES}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \ + | sed -zEn \ + -e '/^((pip|setuptools|wheel)-.+\.dist-info|distutils-precedence\.pth|_distutils_hack|pip|pkg_resources|setuptools|wheel)$/p' \ + | env -C "${PYTHON_SITE_PACKAGES}" xargs -0r \ + rm -rf + ## --- -FROM base +FROM base AS deps SHELL [ "/bin/sh", "-ec" ] COPY /Dockerfile.deps /usr/local/share/ COPY /scripts/* /usr/local/sbin/ -COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ - -COPY --from=apt-gpg /etc/apt/keyrings/ /etc/apt/keyrings/ - COPY /apt/sources.pgdg /etc/apt/sources.list.d/pgdg.sources COPY /apt/preferences.pgdg /etc/apt/preferences.d/pgdg -COPY --from=patroni /usr/local/bin/ /usr/local/bin/ -COPY --from=patroni /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/ +COPY --from=patroni /usr/local/bin/ /usr/local/bin/ +COPY --from=patroni /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/ ## install missing dependencies for Python site-packages RUN f="${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \ @@ -128,33 +80,6 @@ RUN f="${PYTHON_SITE_PACKAGES}/apt-deps.txt" ; \ xargs -a "$f" apt-install.sh ; \ apt-clean.sh -RUN apt-install.sh \ - brotli \ - curl \ - dumb-init \ - file \ - gettext-base \ - gnupg \ - iproute2 \ - iputils-ping \ - jq \ - less \ - libnss-wrapper \ - lsof \ - ncurses-base \ - netbase \ - netcat-openbsd \ - openssl \ - procps \ - psmisc \ - tzdata \ - vim \ - xxd \ - xz-utils \ - zstd \ - ; \ - apt-clean.sh - ## set up locales! RUN _lang=en_US.UTF8 ; \ { \ @@ -174,4 +99,5 @@ RUN _lang=en_US.UTF8 ; \ locale -a | grep -Fixq "${_lang}" ENV LANG=en_US.UTF8 -RUN find /usr/local/sbin/ ! -type d -ls -delete +RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ + jdupes -1LSpr /usr/ diff --git a/apt/preferences.backports b/apt/preferences.backports new file mode 100644 index 0000000..8658903 --- /dev/null +++ b/apt/preferences.backports @@ -0,0 +1,4 @@ +## example: +# Package: src:curl +# Pin: release n=bookworm-backports +# Pin-Priority: 600 diff --git a/apt/sources.debian b/apt/sources.debian index abdd3a2..75c083a 100644 --- a/apt/sources.debian +++ b/apt/sources.debian @@ -1,11 +1,11 @@ Types: deb URIs: http://deb.debian.org/debian -Suites: bookworm bookworm-updates bookworm-proposed-updates -Components: main contrib non-free +Suites: bookworm bookworm-updates bookworm-proposed-updates bookworm-backports +Components: main Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg Types: deb URIs: http://deb.debian.org/debian-security Suites: bookworm-security -Components: main contrib non-free +Components: main Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg diff --git a/build-scripts/image-base.sh b/build-scripts/image-base.sh index 9c1f5db..45323b0 100755 --- a/build-scripts/image-base.sh +++ b/build-scripts/image-base.sh @@ -27,7 +27,7 @@ grab_site_packages() { PYTHON_SITE_PACKAGES=$(grab_site_packages "docker.io/python:${PYTHONTAG}") [ -n "${PYTHON_SITE_PACKAGES:?}" ] -img="docker.io/rockdrilla/postgresql:base-v1" +img="docker.io/rockdrilla/postgresql:base-v2" buildah bud \ -f ./Dockerfile.base \ diff --git a/build-scripts/image-deps.sh b/build-scripts/image-deps.sh index a652720..2a0cb3a 100755 --- a/build-scripts/image-deps.sh +++ b/build-scripts/image-deps.sh @@ -8,9 +8,11 @@ BUILDAH_ISOLATION="${BUILDAH_ISOLATION:-chroot}" BUILDAH_NETWORK="${BUILDAH_NETWORK:-host}" set +a -img="docker.io/rockdrilla/postgresql:deps-v1" +img="docker.io/rockdrilla/postgresql:deps-v2" +base="docker.io/rockdrilla/postgresql:base-v2" exec buildah bud \ -f ./Dockerfile.deps \ -t "${img}" \ - --pull=missing --no-cache + --pull=missing --no-cache \ + --build-arg "BASE_IMAGE=${base}" \ diff --git a/build-scripts/image.sh b/build-scripts/image.sh index c02bcc7..dee9eb2 100755 --- a/build-scripts/image.sh +++ b/build-scripts/image.sh @@ -12,7 +12,7 @@ POSTGRESQL_VERSION="${1:-16.4}" PG_MAJOR="${POSTGRESQL_VERSION%%.*}" img="docker.io/rockdrilla/postgresql:${POSTGRESQL_VERSION}" -deps="docker.io/rockdrilla/postgresql:deps-v1" +deps="docker.io/rockdrilla/postgresql:deps-v2" c=$(buildah from --pull=missing "${deps}") [ -n "${c:?}" ] diff --git a/extra-scripts/certifi-extras.sh b/extra-scripts/certifi-extras.sh index 2e509f7..595f726 100755 --- a/extra-scripts/certifi-extras.sh +++ b/extra-scripts/certifi-extras.sh @@ -1,89 +1,41 @@ #!/bin/sh set -ef -certifi_uri="https://raw.githubusercontent.com/certifi/python-certifi/${CERTIFI_COMMIT:?}/certifi/cacert.pem" dst_dir=/usr/local/share/ca-certificates w=$(mktemp -d) ; : "${w:?}" w_cleanup() { - [ -z "$w" ] || ls -lA "$w/" + [ -z "$w" ] || ls -lA "$w/" >&2 [ -z "$w" ] || rm -rf "$w" unset w exit "${1:-0}" } -curl -sSL "${certifi_uri}" > "$w/certifi.crt" - def_bundle='/etc/ssl/certs/ca-certificates.crt' -openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem" -openssl-cert-auto-pem.sh "$w/certifi.crt" > "$w/certifi.pem" -[ -s "$w/cacert.pem" ] || w_cleanup 1 +openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp" +[ -s "$w/cacert.pem" ] || w_cleanup 1 +[ -s "$w/cacert.fp" ] || w_cleanup 1 + +openssl-cert-auto-pem.sh "$1" "$w/certifi.pem" "$w/certifi.fp" "$w/certifi.off" [ -s "$w/certifi.pem" ] || w_cleanup 1 - -bundle_offsets() { - awk ' - BEGIN { - OFS = "," - m_begin="-----BEGIN CERTIFICATE-----" - m_end="-----END CERTIFICATE-----" - i_begin = 0 - } - $0 == m_begin { i_begin = NR ; } - $0 == m_end { - if (i_begin > 0) { - print i_begin,NR - i_begin = 0 - } - } - ' "$1" -} - -bundle_offsets "$w/cacert.pem" > "$w/cacert.off" -bundle_offsets "$w/certifi.pem" > "$w/certifi.off" -[ -s "$w/cacert.off" ] || w_cleanup 1 +[ -s "$w/certifi.fp" ] || w_cleanup 1 [ -s "$w/certifi.off" ] || w_cleanup 1 -bundle_fingerprints() { - local a - while read -r a ; do - [ -n "$a" ] || continue - - { - sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint -sha256 \ - || \ - sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint - } | tr '[:upper:]' '[:lower:]' - done < "$2" -} - -bundle_fingerprints "$w/cacert.pem" "$w/cacert.off" | sort -uV > "$w/cacert.fp" -bundle_fingerprints "$w/certifi.pem" "$w/certifi.off" | sort -uV > "$w/certifi.fp" -[ -s "$w/cacert.fp" ] || w_cleanup 1 -[ -s "$w/certifi.fp" ] || w_cleanup 1 - set +e -grep -Fxv -f "$w/cacert.fp" "$w/certifi.fp" > "$w/diff.fp" +grep -Fxnv -f "$w/cacert.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/diff.ln" set -e -if [ -s "$w/diff.fp" ] ; then - set +e - grep -Fxn -f "$w/diff.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/records.diff" - set -e - - terse_fingerprint() { - cut -d = -f 2- | tr -cd '[:alnum:]' - } - - mkdir "$w/extras" +if [ -s "$w/diff.ln" ] ; then + terse_fingerprint() { cut -d = -f 2- | tr -cd '[:alnum:]' ; } while read -r n ; do [ -n "$n" ] || continue fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint) off=$(sed -ne "${n}p" "$w/certifi.off") - sed -ne "${off}p" "$w/certifi.pem" | openssl x509 > "${dst_dir}/certifi-${fp}.crt" - done < "$w/records.diff" + sed -ne "${off}p" "$w/certifi.pem" > "${dst_dir}/certifi-${fp}.crt" + done < "$w/diff.ln" fi rm -rf "$w" ; unset w diff --git a/scripts/apt-clean.sh b/scripts/apt-clean.sh index 4365a7c..270b574 100755 --- a/scripts/apt-clean.sh +++ b/scripts/apt-clean.sh @@ -23,24 +23,28 @@ find /var/cache/debconf/ ! -type d -wholename '/var/cache/debconf/*-old' -delete __t=$(mktemp) ; : "${__t:?}" debconf_trim_i18n() { - mawk 'BEGIN { m = 0 } - $0 == "" { print } - /^[^[:space:]]/ { - if ($1 ~ "\.[Uu][Tt][Ff]-?8:") { m = 1; next; } - m = 0; print $0; - } - /^[[:space:]]/ { - if (m == 1) next; - print $0; - }' < "$1" > "${__t}" - cat < "${__t}" > "$1" + mawk 'BEGIN { m = 0; } + $0 == "" { print; } + /^[^[:space:]]/ { + if ($1 ~ "\.[Uu][Tt][Ff]-?8:") { + m = 1; + next; + } + m = 0; + print $0; + } + /^[[:space:]]/ { + if (m == 1) next; + print $0; + }' < "$1" > "${__t}" + cat < "${__t}" > "$1" } debconf_trim_i18n /var/cache/debconf/templates.dat while read -r tmpl ; do - [ -n "${tmpl}" ] || continue - [ -s "${tmpl}" ] || continue - debconf_trim_i18n "${tmpl}" + [ -n "${tmpl}" ] || continue + [ -s "${tmpl}" ] || continue + debconf_trim_i18n "${tmpl}" done <&2 [ -z "$w" ] || rm -rf "$w" unset w exit "${1:-0}" } +bundle_offsets() { + mawk 'BEGIN { OFS = ","; i_begin = 0; } + $0 == "-----BEGIN CERTIFICATE-----" { + i_begin = NR; + } + $0 == "-----END CERTIFICATE-----" { + if (i_begin > 0) { + print i_begin, NR; + i_begin = 0; + } + }' "$1" +} + +bundle_fingerprints() { + local x f + while read -r x ; do + [ -n "$x" ] || continue + + f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint -sha256) + [ -n "$f" ] || f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint) + [ -n "$f" ] || continue + + printf '%s\n' "$f" | tr '[:upper:]' '[:lower:]' + done < "$2" +} + openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1 [ -s "$w/cert.pem" ] || w_cleanup 1 tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt" [ -s "$w/cert.txt" ] || w_cleanup 1 +rm -f "$w/cert.pem" -awk ' -BEGIN { - OFS = "," - m_begin="-----BEGIN CERTIFICATE-----" - m_end="-----END CERTIFICATE-----" - i_begin = 0 -} -$0 == m_begin { i_begin = NR ; } -$0 == m_end { - if (i_begin > 0) { - print i_begin,NR - i_begin = 0 - } -} -' "$w/cert.txt" > "$w/cert.offsets" -[ -s "$w/cert.offsets" ] || w_cleanup 1 +bundle_offsets "$w/cert.txt" > "$w/cert.off" +[ -s "$w/cert.off" ] || w_cleanup 1 -while read -r a ; do - [ -n "$a" ] || continue +bundle_fingerprints "$w/cert.txt" "$w/cert.off" > "$w/cert.fp.all" +[ -s "$w/cert.fp.all" ] || w_cleanup 1 - sed -ne "${a}p" "$w/cert.txt" -done < "$w/cert.offsets" +sort -uV < "$w/cert.fp.all" > "$w/cert.fp" +while read -r fp ; do + [ -n "${fp}" ] || continue + + n=$(grep -m1 -Fxn -e "${fp}" "$w/cert.fp.all" | cut -d : -f 1) + [ -n "$n" ] || continue + + off=$(sed -ne "${n}p" "$w/cert.off") + [ -n "${off}" ] || continue + + sed -ne "${off}p" "$w/cert.txt" +done < "$w/cert.fp" > "$w/cert.pem" +[ -s "$w/cert.pem" ] || w_cleanup 1 +rm -f "$w/cert.txt" "$w/cert.off" "$w/cert.fp.all" + +if [ -n "$2" ] ; then + while : ; do + if [ -e "$2" ] ; then + [ -f "$2" ] || break + fi + cat > "$2" + break ; done +else + cat +fi < "$w/cert.pem" + +while [ -n "$3" ] ; do + if [ -e "$3" ] ; then + [ -f "$3" ] || break + fi + cat "$w/cert.fp" > "$3" +break ; done + +while [ -n "$4" ] ; do + if [ -e "$4" ] ; then + [ -f "$4" ] || break + fi + bundle_offsets "$w/cert.pem" > "$4" +break ; done rm -rf "$w" ; unset w diff --git a/scripts/openssl-cert-fingerprint.sh b/scripts/openssl-cert-fingerprint.sh deleted file mode 100755 index f41ad1d..0000000 --- a/scripts/openssl-cert-fingerprint.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -set -f - -[ $# -gt 0 ] || exit 0 -me=${0##*/} - -[ -n "$1" ] || exit 1 -[ -f "$1" ] || { - env printf '%s: not a file or does not exist: %q\n' "${me}" "$1" >&2 - exit 1 -} -[ -s "$1" ] || exit 0 - -w=$(mktemp -d) || exit 1 -w_cleanup() { - [ -z "$w" ] || ls -lA "$w/" - [ -z "$w" ] || rm -rf "$w" - unset w - exit "${1:-0}" -} - -openssl-cert-auto-pem.sh "$1" > "$w/cert.pem" || w_cleanup 1 -[ -s "$w/cert.pem" ] || w_cleanup 1 - -awk ' -BEGIN { - OFS = "," - m_begin="-----BEGIN CERTIFICATE-----" - m_end="-----END CERTIFICATE-----" - i_begin = 0 -} -$0 == m_begin { i_begin = NR ; } -$0 == m_end { - if (i_begin > 0) { - print i_begin,NR - i_begin = 0 - } -} -' "$w/cert.pem" > "$w/cert.off" -[ -s "$w/cert.off" ] || w_cleanup 1 - -while read -r a ; do - [ -n "$a" ] || continue - - { - sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint -sha256 \ - || \ - sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint - } | tr '[:upper:]' '[:lower:]' -done < "$w/cert.off" - -w_cleanup 0 diff --git a/scripts/python-rm-cache.sh b/scripts/python-rm-cache.sh index 039b846..a1b887e 100755 --- a/scripts/python-rm-cache.sh +++ b/scripts/python-rm-cache.sh @@ -1,7 +1,9 @@ #!/bin/sh set -f for i ; do - find "$i/" -name __pycache__ -exec rm -rf {} + - find "$i/" ! -type d -name '*.py[co]' -exec rm -f {} + + [ -n "$i" ] || continue + [ -d "$i" ] || continue + find "$i/" -name __pycache__ -exec rm -rf {} + + find "$i/" ! -type d -name '*.py[co]' -exec rm -f {} + done exit 0