From 91ec2a4c863c31d4c4e8039ce3980325176e560f Mon Sep 17 00:00:00 2001 From: Konstantin Demin Date: Fri, 20 Sep 2024 02:39:40 +0300 Subject: [PATCH] treewide: rework certificate handling also: - reorder some blocks in dockerfiles - provide sane requirements.txt --- Dockerfile | 30 ++++----- Dockerfile.base | 36 +++++++++- Dockerfile.deps | 26 ++------ extra-scripts/certifi-extras.sh | 72 ++++---------------- image-entry.d/77-openssl-ca-certs.envsh | 69 ++++++++++---------- requirements.txt | 5 ++ scripts/openssl-cert-auto-pem.sh | 87 +++++++++++++++++++------ scripts/openssl-cert-fingerprint.sh | 52 --------------- 8 files changed, 174 insertions(+), 203 deletions(-) create mode 100644 requirements.txt delete mode 100755 scripts/openssl-cert-fingerprint.sh diff --git a/Dockerfile b/Dockerfile index 98d88e8..33fbfbb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,32 +3,31 @@ FROM docker.io/rockdrilla/angie-conv:${IMAGE_VERSION}-deps AS deps ## --- -FROM deps AS certs +FROM deps AS setup SHELL [ "/bin/sh", "-ec" ] COPY /scripts/* /usr/local/sbin/ COPY /extra-scripts/* /usr/local/sbin/ -## consult https://github.com/certifi/python-certifi/ -ENV CERTIFI_COMMIT=bd8153872e9c6fc98f4023df9c2deaffea2fa463 +ADD https://angie.software/keys/angie-signing.gpg /tmp/angie.gpg.bin +COPY /apt/sources.angie /etc/apt/sources.list.d/angie.txt -RUN apt-install.sh ca-certificates ; \ - ## process certifi - ca_file='/etc/ssl/certs/ca-certificates.crt' ; \ - openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp.orig" ; \ - ls -l "${ca_file}" ; \ - certifi-extras.sh ; \ - openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp" ; \ - chmod 0444 "${ca_file}" "${ca_file}.fp" "${ca_file}.fp.orig" ; \ - ls -l "${ca_file}" "${ca_file}.fp" "${ca_file}.fp.orig" +RUN pkg='gnupg' ; \ + apt-install.sh ${pkg} ; \ + ## process Angie GPG keyring / APT sources + gpg-export.sh /tmp/angie.gpg.bin /etc/apt/keyrings/angie.gpg.asc ; \ + rm -f /tmp/angie.gpg.bin ; \ + env -C /etc/apt/sources.list.d mv angie.txt angie.sources ; \ + ## verify sources! + apt-env.sh apt-get update ; \ + apt-remove.sh ${pkg} ; \ + apt-clean.sh ## --- FROM deps AS pycache SHELL [ "/bin/sh", "-ec" ] -COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ - COPY /scripts/* /usr/local/sbin/ COPY /extra-scripts/* /usr/local/sbin/ @@ -74,7 +73,8 @@ SHELL [ "/bin/sh", "-ec" ] COPY /Dockerfile /usr/local/share/ -COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ +COPY --from=setup /etc/apt/keyrings/angie.gpg.asc /etc/apt/keyrings/ +COPY --from=setup /etc/apt/sources.list.d/angie.sources /etc/apt/sources.list.d/ ## RFC: Python cache ## TODO: reduce load by selecting only __pycache__ directories in either way diff --git a/Dockerfile.base b/Dockerfile.base index f8ca9f4..1b471c1 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -2,7 +2,7 @@ ARG PYTHONTAG=3.11.10-slim-bookworm FROM docker.io/python:${PYTHONTAG} AS base-upstream -FROM base-upstream AS base +FROM base-upstream AS base-intermediate SHELL [ "/bin/sh", "-ec" ] COPY /Dockerfile.base /usr/local/share/ @@ -223,5 +223,39 @@ RUN find /usr/local/sbin/ ! -type d -ls -delete ; \ find /run/ -mindepth 1 -ls -delete || : ; \ install -d -m 01777 /run/lock +## --- + +FROM base-intermediate AS certs +SHELL [ "/bin/sh", "-ec" ] + +COPY /scripts/* /usr/local/sbin/ +COPY /extra-scripts/* /usr/local/sbin/ + +## "2024.08.30" +ENV CERTIFI_COMMIT=325c2fde4f8eec10d682b09f3b0414dc05e69a81 + +# 'https://raw.githubusercontent.com/certifi/python-certifi' +ARG CERTIFI_BASE_URI='https://github.com/certifi/python-certifi/raw' + +ARG CERTIFI_URI="${CERTIFI_BASE_URI}/${CERTIFI_COMMIT}/certifi/cacert.pem" +ADD "${CERTIFI_URI}" /tmp/certifi.crt + +RUN apt-install.sh ca-certificates ; \ + apt-clean.sh ; \ + ca_file='/etc/ssl/certs/ca-certificates.crt' ; \ + ls -l "${ca_file}" ; \ + ## process certifi + certifi-extras.sh /tmp/certifi.crt ; \ + openssl-cert-auto-pem.sh "${ca_file}" "${ca_file}.new" "${ca_file}.fp" ; \ + mv -f "${ca_file}.new" "${ca_file}" ; \ + chmod 0444 "${ca_file}" "${ca_file}.fp" ; \ + ls -l "${ca_file}" "${ca_file}.fp" + +## --- + +FROM base-intermediate AS base + +COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/ + ENTRYPOINT [ ] CMD [ "bash" ] diff --git a/Dockerfile.deps b/Dockerfile.deps index 2cee13d..72db77f 100644 --- a/Dockerfile.deps +++ b/Dockerfile.deps @@ -3,27 +3,14 @@ FROM docker.io/rockdrilla/angie-conv:${IMAGE_VERSION}-base AS base ## --- -FROM base AS setup +FROM base AS build SHELL [ "/bin/sh", "-ec" ] COPY /scripts/* /usr/local/sbin/ COPY /extra-scripts/* /usr/local/sbin/ -ADD https://angie.software/keys/angie-signing.gpg /tmp/angie.gpg.bin -COPY /apt/sources.angie /etc/apt/sources.list.d/angie.txt +COPY /requirements.txt /tmp/ -RUN pkg='gnupg' ; \ - apt-install.sh ${pkg} ; \ - ## process Angie GPG keyring / APT sources - gpg-export.sh /tmp/angie.gpg.bin /etc/apt/keyrings/angie.gpg.asc ; \ - rm -f /tmp/angie.gpg.bin ; \ - env -C /etc/apt/sources.list.d mv angie.txt angie.sources ; \ - ## verify sources! - apt-env.sh apt-get update ; \ - apt-remove.sh ${pkg} ; \ - apt-clean.sh - -ENV INSTALL_WHEELS='jinja2 netaddr psutil pyyaml wcmatch' ENV DEV_PACKAGES='libyaml-dev' # markupsafe, psutil ENV CIBUILDWHEEL=1 @@ -43,7 +30,7 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \ rm -rf "$w/" ; unset w ; \ apt-install.sh build-essential ; \ pip-env.sh pip install 'cython' ; \ - pip-env.sh pip install --no-binary :all: ${INSTALL_WHEELS} ; \ + pip-env.sh pip install --no-binary :all: -r /tmp/requirements.txt ; \ pip-env.sh pip uninstall -y 'cython' ; \ python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" ; \ rm -rf \ @@ -67,12 +54,9 @@ SHELL [ "/bin/sh", "-ec" ] COPY /Dockerfile.deps /usr/local/share/ -COPY --from=setup /etc/apt/keyrings/angie.gpg.asc /etc/apt/keyrings/ -COPY --from=setup /etc/apt/sources.list.d/angie.sources /etc/apt/sources.list.d/ - ## Python: site-packages -COPY --from=setup /usr/local/bin/ /usr/local/bin/ -COPY --from=setup /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/ +COPY --from=build /usr/local/bin/ /usr/local/bin/ +COPY --from=build /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/ COPY /scripts/* /usr/local/sbin/ diff --git a/extra-scripts/certifi-extras.sh b/extra-scripts/certifi-extras.sh index 2e509f7..595f726 100755 --- a/extra-scripts/certifi-extras.sh +++ b/extra-scripts/certifi-extras.sh @@ -1,89 +1,41 @@ #!/bin/sh set -ef -certifi_uri="https://raw.githubusercontent.com/certifi/python-certifi/${CERTIFI_COMMIT:?}/certifi/cacert.pem" dst_dir=/usr/local/share/ca-certificates w=$(mktemp -d) ; : "${w:?}" w_cleanup() { - [ -z "$w" ] || ls -lA "$w/" + [ -z "$w" ] || ls -lA "$w/" >&2 [ -z "$w" ] || rm -rf "$w" unset w exit "${1:-0}" } -curl -sSL "${certifi_uri}" > "$w/certifi.crt" - def_bundle='/etc/ssl/certs/ca-certificates.crt' -openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem" -openssl-cert-auto-pem.sh "$w/certifi.crt" > "$w/certifi.pem" -[ -s "$w/cacert.pem" ] || w_cleanup 1 +openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp" +[ -s "$w/cacert.pem" ] || w_cleanup 1 +[ -s "$w/cacert.fp" ] || w_cleanup 1 + +openssl-cert-auto-pem.sh "$1" "$w/certifi.pem" "$w/certifi.fp" "$w/certifi.off" [ -s "$w/certifi.pem" ] || w_cleanup 1 - -bundle_offsets() { - awk ' - BEGIN { - OFS = "," - m_begin="-----BEGIN CERTIFICATE-----" - m_end="-----END CERTIFICATE-----" - i_begin = 0 - } - $0 == m_begin { i_begin = NR ; } - $0 == m_end { - if (i_begin > 0) { - print i_begin,NR - i_begin = 0 - } - } - ' "$1" -} - -bundle_offsets "$w/cacert.pem" > "$w/cacert.off" -bundle_offsets "$w/certifi.pem" > "$w/certifi.off" -[ -s "$w/cacert.off" ] || w_cleanup 1 +[ -s "$w/certifi.fp" ] || w_cleanup 1 [ -s "$w/certifi.off" ] || w_cleanup 1 -bundle_fingerprints() { - local a - while read -r a ; do - [ -n "$a" ] || continue - - { - sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint -sha256 \ - || \ - sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint - } | tr '[:upper:]' '[:lower:]' - done < "$2" -} - -bundle_fingerprints "$w/cacert.pem" "$w/cacert.off" | sort -uV > "$w/cacert.fp" -bundle_fingerprints "$w/certifi.pem" "$w/certifi.off" | sort -uV > "$w/certifi.fp" -[ -s "$w/cacert.fp" ] || w_cleanup 1 -[ -s "$w/certifi.fp" ] || w_cleanup 1 - set +e -grep -Fxv -f "$w/cacert.fp" "$w/certifi.fp" > "$w/diff.fp" +grep -Fxnv -f "$w/cacert.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/diff.ln" set -e -if [ -s "$w/diff.fp" ] ; then - set +e - grep -Fxn -f "$w/diff.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/records.diff" - set -e - - terse_fingerprint() { - cut -d = -f 2- | tr -cd '[:alnum:]' - } - - mkdir "$w/extras" +if [ -s "$w/diff.ln" ] ; then + terse_fingerprint() { cut -d = -f 2- | tr -cd '[:alnum:]' ; } while read -r n ; do [ -n "$n" ] || continue fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint) off=$(sed -ne "${n}p" "$w/certifi.off") - sed -ne "${off}p" "$w/certifi.pem" | openssl x509 > "${dst_dir}/certifi-${fp}.crt" - done < "$w/records.diff" + sed -ne "${off}p" "$w/certifi.pem" > "${dst_dir}/certifi-${fp}.crt" + done < "$w/diff.ln" fi rm -rf "$w" ; unset w diff --git a/image-entry.d/77-openssl-ca-certs.envsh b/image-entry.d/77-openssl-ca-certs.envsh index 2b4333b..861d5e8 100755 --- a/image-entry.d/77-openssl-ca-certs.envsh +++ b/image-entry.d/77-openssl-ca-certs.envsh @@ -30,37 +30,43 @@ while : ; do unset orig_ca_file [ -s "$w/all.pem" ] || break - openssl-cert-fingerprint.sh "$w/all.pem" | sort -uV > "$w/all.fp" - [ -s "$w/all.fp" ] || break + openssl-cert-auto-pem.sh "$w/all.pem" "$w/new.pem" "$w/new.fp" "$w/new.off" + [ -s "$w/new.pem" ] || break + [ -s "$w/new.fp" ] || break + [ -s "$w/new.off" ] || break + rm -f "$w/all.pem" ## leaving processing section rm -f "$w/processing" - unset dev_root dev_bundle dev_bundle_fp - dev_root=$(env stat -c '%d' / ) - dev_bundle=$(env stat -L -c '%d' "${def_bundle}") - dev_bundle_fp=$(env stat -L -c '%d' "${def_bundle_fp}") - unset def_bundle_bind_mount def_bundle_bind_mount=1 while : ; do - [ "${dev_root}" = "${dev_bundle}" ] || break - [ "${dev_root}" = "${dev_bundle_fp}" ] || break - [ "${dev_bundle}" = "${dev_bundle_fp}" ] || break + unset devno_root devno_bundle devno_bundle_fp + devno_root=$(env stat -c '%d' / ) + + [ -f "${def_bundle}" ] || break + devno_bundle=$(env stat -L -c '%d' "${def_bundle}") + [ "${devno_root}" = "${devno_bundle}" ] || break + + [ -f "${def_bundle_fp}" ] || break + devno_bundle_fp=$(env stat -L -c '%d' "${def_bundle_fp}") + [ "${devno_root}" = "${devno_bundle_fp}" ] || break def_bundle_bind_mount=0 break ; done - unset dev_root dev_bundle dev_bundle_fp + unset devno_root devno_bundle devno_bundle_fp if [ "${def_bundle_bind_mount}" = 1 ] ; then log_always "detected bind-mount inside ${def_bundle%/*}/" log_always "this is merely misuse!" - openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem" - openssl-cert-fingerprint.sh "$w/cacert.pem" | sort -uV > "$w/cacert.fp" + if [ -s "${def_bundle}" ] ; then + openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp" + fi else - ln -s "${def_bundle}" "$w/cacert.pem" - ln -s "${def_bundle_fp}" "$w/cacert.fp" + ln -s "${def_bundle}" "$w/cacert.pem" + ln -s "${def_bundle_fp}" "$w/cacert.fp" fi unset with_def_bundle @@ -73,19 +79,10 @@ while : ; do break ; done if [ "${with_def_bundle}" = 1 ] ; then - grep -Fxv -f "$w/cacert.fp" "$w/all.fp" > "$w/diff.fp" - [ -s "$w/diff.fp" ] || break - - ## entering processing section - touch "$w/processing" - - grep -Fxn -f "$w/diff.fp" "$w/all.fp" | cut -d : -f 1 > "$w/diff.lineno" - [ -s "$w/diff.lineno" ] || break - - ## leaving processing section - rm -f "$w/processing" + grep -Fxnv -f "$w/cacert.fp" "$w/new.fp" | cut -d : -f 1 > "$w/diff.ln" + [ -s "$w/diff.ln" ] || break else - : > "$w/diff.lineno" + : > "$w/diff.ln" fi : > "${volume_root}/ca.pem" @@ -99,9 +96,11 @@ while : ; do while read -r n ; do [ -n "$n" ] || continue - off=$(sed -ne "${n}p" "$w/all.off") - sed -ne "${off}p" "$w/all.pem" | openssl x509 - done < "$w/diff.lineno" >> "${volume_root}/ca.pem" + off=$(sed -ne "${n}p" "$w/new.off") + [ -n "${off}" ] || continue + + sed -ne "${off}p" "$w/new.pem" + done < "$w/diff.ln" >> "${volume_root}/ca.pem" unset n off set -a @@ -110,10 +109,14 @@ while : ; do SSL_CERT_DIR="${empty_dir}" set +a break ; done -unset def_bundle_bind_mount with_def_bundle +unset def_bundle_fp def_bundle_bind_mount with_def_bundle -[ -f "${volume_root}/ca.pem" ] || ln -s "${def_bundle}" "${volume_root}/ca.pem" -unset def_bundle def_bundle_fp +while ! [ -f "${volume_root}/ca.pem" ] ; do + [ -s "${def_bundle}" ] || break + ln -s "${def_bundle}" "${volume_root}/ca.pem" +break ; done +unset def_bundle +[ -f "${volume_root}/ca.pem" ] || : > "${volume_root}/ca.pem" if [ -n "${w:-}" ] ; then if [ -f "$w/processing" ] ; then diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f3a7bd5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +jinja2==3.1.4 +netaddr==1.3.0 +psutil==6.0.0 +pyyaml==6.0.2 +wcmatch==9.0 diff --git a/scripts/openssl-cert-auto-pem.sh b/scripts/openssl-cert-auto-pem.sh index 2ffff27..3487e88 100755 --- a/scripts/openssl-cert-auto-pem.sh +++ b/scripts/openssl-cert-auto-pem.sh @@ -13,38 +13,83 @@ me=${0##*/} w=$(mktemp -d) || exit 1 w_cleanup() { - [ -z "$w" ] || ls -lA "$w/" + [ -z "$w" ] || ls -lA "$w/" >&2 [ -z "$w" ] || rm -rf "$w" unset w exit "${1:-0}" } +bundle_offsets() { + awk ' + BEGIN { OFS = "," ; i_begin = 0 ; } + $0 == "-----BEGIN CERTIFICATE-----" { i_begin = NR ; } + $0 == "-----END CERTIFICATE-----" { if (i_begin > 0) { print i_begin,NR ; i_begin = 0 ; } } + ' "$1" +} + +bundle_fingerprints() { + local x f + while read -r x ; do + [ -n "$x" ] || continue + + f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint -sha256) + [ -n "$f" ] || f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint) + [ -n "$f" ] || continue + + printf '%s\n' "$f" | tr '[:upper:]' '[:lower:]' + done < "$2" +} + openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1 [ -s "$w/cert.pem" ] || w_cleanup 1 tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt" [ -s "$w/cert.txt" ] || w_cleanup 1 +rm -f "$w/cert.pem" -awk ' -BEGIN { - OFS = "," - m_begin="-----BEGIN CERTIFICATE-----" - m_end="-----END CERTIFICATE-----" - i_begin = 0 -} -$0 == m_begin { i_begin = NR ; } -$0 == m_end { - if (i_begin > 0) { - print i_begin,NR - i_begin = 0 - } -} -' "$w/cert.txt" > "$w/cert.offsets" -[ -s "$w/cert.offsets" ] || w_cleanup 1 +bundle_offsets "$w/cert.txt" > "$w/cert.off" +[ -s "$w/cert.off" ] || w_cleanup 1 -while read -r a ; do - [ -n "$a" ] || continue +bundle_fingerprints "$w/cert.txt" "$w/cert.off" > "$w/cert.fp.all" +[ -s "$w/cert.fp.all" ] || w_cleanup 1 - sed -ne "${a}p" "$w/cert.txt" -done < "$w/cert.offsets" +sort -uV < "$w/cert.fp.all" > "$w/cert.fp" +while read -r fp ; do + [ -n "${fp}" ] || continue + + n=$(grep -m1 -Fxn -e "${fp}" "$w/cert.fp.all" | cut -d : -f 1) + [ -n "$n" ] || continue + + off=$(sed -ne "${n}p" "$w/cert.off") + [ -n "${off}" ] || continue + + sed -ne "${off}p" "$w/cert.txt" +done < "$w/cert.fp" > "$w/cert.pem" +[ -s "$w/cert.pem" ] || w_cleanup 1 +rm -f "$w/cert.txt" "$w/cert.off" "$w/cert.fp.all" + +if [ -n "$2" ] ; then + while : ; do + if [ -e "$2" ] ; then + [ -f "$2" ] || break + fi + cat > "$2" + break ; done +else + cat +fi < "$w/cert.pem" + +while [ -n "$3" ] ; do + if [ -e "$3" ] ; then + [ -f "$3" ] || break + fi + cat "$w/cert.fp" > "$3" +break ; done + +while [ -n "$4" ] ; do + if [ -e "$4" ] ; then + [ -f "$4" ] || break + fi + bundle_offsets "$w/cert.pem" > "$4" +break ; done rm -rf "$w" ; unset w diff --git a/scripts/openssl-cert-fingerprint.sh b/scripts/openssl-cert-fingerprint.sh deleted file mode 100755 index f41ad1d..0000000 --- a/scripts/openssl-cert-fingerprint.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -set -f - -[ $# -gt 0 ] || exit 0 -me=${0##*/} - -[ -n "$1" ] || exit 1 -[ -f "$1" ] || { - env printf '%s: not a file or does not exist: %q\n' "${me}" "$1" >&2 - exit 1 -} -[ -s "$1" ] || exit 0 - -w=$(mktemp -d) || exit 1 -w_cleanup() { - [ -z "$w" ] || ls -lA "$w/" - [ -z "$w" ] || rm -rf "$w" - unset w - exit "${1:-0}" -} - -openssl-cert-auto-pem.sh "$1" > "$w/cert.pem" || w_cleanup 1 -[ -s "$w/cert.pem" ] || w_cleanup 1 - -awk ' -BEGIN { - OFS = "," - m_begin="-----BEGIN CERTIFICATE-----" - m_end="-----END CERTIFICATE-----" - i_begin = 0 -} -$0 == m_begin { i_begin = NR ; } -$0 == m_end { - if (i_begin > 0) { - print i_begin,NR - i_begin = 0 - } -} -' "$w/cert.pem" > "$w/cert.off" -[ -s "$w/cert.off" ] || w_cleanup 1 - -while read -r a ; do - [ -n "$a" ] || continue - - { - sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint -sha256 \ - || \ - sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint - } | tr '[:upper:]' '[:lower:]' -done < "$w/cert.off" - -w_cleanup 0