treewide: rework certificate handling

also:
- reorder some blocks in dockerfiles
- provide sane requirements.txt

Dockerfile

@@ -3,32 +3,31 @@ FROM docker.io/rockdrilla/angie-conv:${IMAGE_VERSION}-deps AS deps
 
 ## ---
 
-FROM deps AS certs
+FROM deps AS setup
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /scripts/*       /usr/local/sbin/
 COPY /extra-scripts/* /usr/local/sbin/
 
-## consult https://github.com/certifi/python-certifi/
+ADD https://angie.software/keys/angie-signing.gpg /tmp/angie.gpg.bin
-ENV CERTIFI_COMMIT=bd8153872e9c6fc98f4023df9c2deaffea2fa463
+COPY /apt/sources.angie /etc/apt/sources.list.d/angie.txt
 
-RUN apt-install.sh ca-certificates ; \
+RUN pkg='gnupg' ; \
-    ## process certifi
+    apt-install.sh ${pkg} ; \
-    ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
+    ## process Angie GPG keyring / APT sources
-    openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp.orig" ; \
+    gpg-export.sh /tmp/angie.gpg.bin /etc/apt/keyrings/angie.gpg.asc ; \
-    ls -l "${ca_file}" ; \
+    rm -f /tmp/angie.gpg.bin ; \
-    certifi-extras.sh ; \
+    env -C /etc/apt/sources.list.d mv angie.txt angie.sources ; \
-    openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp" ; \
+    ## verify sources!
-    chmod 0444 "${ca_file}" "${ca_file}.fp" "${ca_file}.fp.orig" ; \
+    apt-env.sh apt-get update ; \
-    ls -l "${ca_file}" "${ca_file}.fp" "${ca_file}.fp.orig"
+    apt-remove.sh ${pkg} ; \
+    apt-clean.sh
 
 ## ---
 
 FROM deps AS pycache
 SHELL [ "/bin/sh", "-ec" ]
 
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt  /etc/ssl/certs/
-
 COPY /scripts/*       /usr/local/sbin/
 COPY /extra-scripts/* /usr/local/sbin/
 
@@ -74,7 +73,8 @@ SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile /usr/local/share/
 
-COPY --from=certs /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
+COPY --from=setup /etc/apt/keyrings/angie.gpg.asc        /etc/apt/keyrings/
+COPY --from=setup /etc/apt/sources.list.d/angie.sources  /etc/apt/sources.list.d/
 
 ## RFC: Python cache
 ## TODO: reduce load by selecting only __pycache__ directories in either way

Dockerfile.base

@@ -2,7 +2,7 @@
 ARG PYTHONTAG=3.11.10-slim-bookworm
 FROM docker.io/python:${PYTHONTAG} AS base-upstream
 
-FROM base-upstream AS base
+FROM base-upstream AS base-intermediate
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile.base /usr/local/share/
@@ -223,5 +223,39 @@ RUN find /usr/local/sbin/ ! -type d -ls -delete ; \
     find /run/ -mindepth 1 -ls -delete || : ; \
     install -d -m 01777 /run/lock
 
+## ---
+
+FROM base-intermediate AS certs
+SHELL [ "/bin/sh", "-ec" ]
+
+COPY /scripts/*       /usr/local/sbin/
+COPY /extra-scripts/* /usr/local/sbin/
+
+## "2024.08.30"
+ENV CERTIFI_COMMIT=325c2fde4f8eec10d682b09f3b0414dc05e69a81
+
+# 'https://raw.githubusercontent.com/certifi/python-certifi'
+ARG CERTIFI_BASE_URI='https://github.com/certifi/python-certifi/raw'
+
+ARG CERTIFI_URI="${CERTIFI_BASE_URI}/${CERTIFI_COMMIT}/certifi/cacert.pem"
+ADD "${CERTIFI_URI}" /tmp/certifi.crt
+
+RUN apt-install.sh ca-certificates ; \
+    apt-clean.sh ; \
+    ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
+    ls -l "${ca_file}" ; \
+    ## process certifi
+    certifi-extras.sh /tmp/certifi.crt ; \
+    openssl-cert-auto-pem.sh "${ca_file}" "${ca_file}.new" "${ca_file}.fp" ; \
+    mv -f "${ca_file}.new" "${ca_file}" ; \
+    chmod 0444 "${ca_file}" "${ca_file}.fp" ; \
+    ls -l "${ca_file}" "${ca_file}.fp"
+
+## ---
+
+FROM base-intermediate AS base
+
+COPY --from=certs /etc/ssl/certs/ca-certificates.*  /etc/ssl/certs/
+
 ENTRYPOINT [ ]
 CMD [ "bash" ]

Dockerfile.deps

@@ -3,27 +3,14 @@ FROM docker.io/rockdrilla/angie-conv:${IMAGE_VERSION}-base AS base
 
 ## ---
 
-FROM base AS setup
+FROM base AS build
 SHELL [ "/bin/sh", "-ec" ]
 
 COPY /scripts/*       /usr/local/sbin/
 COPY /extra-scripts/* /usr/local/sbin/
 
-ADD https://angie.software/keys/angie-signing.gpg /tmp/angie.gpg.bin
+COPY /requirements.txt /tmp/
-COPY /apt/sources.angie /etc/apt/sources.list.d/angie.txt
 
-RUN pkg='gnupg' ; \
-    apt-install.sh ${pkg} ; \
-    ## process Angie GPG keyring / APT sources
-    gpg-export.sh /tmp/angie.gpg.bin /etc/apt/keyrings/angie.gpg.asc ; \
-    rm -f /tmp/angie.gpg.bin ; \
-    env -C /etc/apt/sources.list.d mv angie.txt angie.sources ; \
-    ## verify sources!
-    apt-env.sh apt-get update ; \
-    apt-remove.sh ${pkg} ; \
-    apt-clean.sh
-
-ENV INSTALL_WHEELS='jinja2 netaddr psutil pyyaml wcmatch'
 ENV DEV_PACKAGES='libyaml-dev'
 # markupsafe, psutil
 ENV CIBUILDWHEEL=1
@@ -43,7 +30,7 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \
     rm -rf "$w/" ; unset w ; \
     apt-install.sh build-essential ; \
     pip-env.sh pip install 'cython' ; \
-    pip-env.sh pip install --no-binary :all: ${INSTALL_WHEELS} ; \
+    pip-env.sh pip install --no-binary :all: -r /tmp/requirements.txt ; \
     pip-env.sh pip uninstall -y 'cython' ; \
     python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" ; \
     rm -rf \
@@ -67,12 +54,9 @@ SHELL [ "/bin/sh", "-ec" ]
 
 COPY /Dockerfile.deps /usr/local/share/
 
-COPY --from=setup /etc/apt/keyrings/angie.gpg.asc        /etc/apt/keyrings/
-COPY --from=setup /etc/apt/sources.list.d/angie.sources  /etc/apt/sources.list.d/
-
 ## Python: site-packages
-COPY --from=setup /usr/local/bin/            /usr/local/bin/
+COPY --from=build /usr/local/bin/            /usr/local/bin/
-COPY --from=setup /${PYTHON_SITE_PACKAGES}/  /${PYTHON_SITE_PACKAGES}/
+COPY --from=build /${PYTHON_SITE_PACKAGES}/  /${PYTHON_SITE_PACKAGES}/
 
 COPY /scripts/* /usr/local/sbin/
 

extra-scripts/certifi-extras.sh

@@ -1,89 +1,41 @@
 #!/bin/sh
 set -ef
 
-certifi_uri="https://raw.githubusercontent.com/certifi/python-certifi/${CERTIFI_COMMIT:?}/certifi/cacert.pem"
 dst_dir=/usr/local/share/ca-certificates
 
 w=$(mktemp -d) ; : "${w:?}"
 w_cleanup() {
-	[ -z "$w" ] || ls -lA "$w/"
+	[ -z "$w" ] || ls -lA "$w/" >&2
 	[ -z "$w" ] || rm -rf "$w"
 	unset w
 	exit "${1:-0}"
 }
 
-curl -sSL "${certifi_uri}" > "$w/certifi.crt"
-
 def_bundle='/etc/ssl/certs/ca-certificates.crt'
 
-openssl-cert-auto-pem.sh "${def_bundle}"  > "$w/cacert.pem"
+openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp"
-openssl-cert-auto-pem.sh "$w/certifi.crt" > "$w/certifi.pem"
+[ -s "$w/cacert.pem" ] || w_cleanup 1
-[ -s "$w/cacert.pem" ]  || w_cleanup 1
+[ -s "$w/cacert.fp" ]  || w_cleanup 1
+
+openssl-cert-auto-pem.sh "$1" "$w/certifi.pem" "$w/certifi.fp" "$w/certifi.off"
 [ -s "$w/certifi.pem" ] || w_cleanup 1
-
+[ -s "$w/certifi.fp" ]  || w_cleanup 1
-bundle_offsets() {
-	awk '
-	BEGIN {
-	    OFS = ","
-	    m_begin="-----BEGIN CERTIFICATE-----"
-	    m_end="-----END CERTIFICATE-----"
-	    i_begin = 0
-	}
-	$0 == m_begin { i_begin = NR ; }
-	$0 == m_end {
-	    if (i_begin > 0) {
-	        print i_begin,NR
-	        i_begin = 0
-	    }
-	}
-	' "$1"
-}
-
-bundle_offsets "$w/cacert.pem"  > "$w/cacert.off"
-bundle_offsets "$w/certifi.pem" > "$w/certifi.off"
-[ -s "$w/cacert.off" ]  || w_cleanup 1
 [ -s "$w/certifi.off" ] || w_cleanup 1
 
-bundle_fingerprints() {
-	local a
-	while read -r a ; do
-		[ -n "$a" ] || continue
-
-		{
-			sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint -sha256 \
-			|| \
-			sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint
-		} | tr '[:upper:]' '[:lower:]'
-	done < "$2"
-}
-
-bundle_fingerprints "$w/cacert.pem"  "$w/cacert.off"  | sort -uV > "$w/cacert.fp"
-bundle_fingerprints "$w/certifi.pem" "$w/certifi.off" | sort -uV > "$w/certifi.fp"
-[ -s "$w/cacert.fp" ]  || w_cleanup 1
-[ -s "$w/certifi.fp" ] || w_cleanup 1
-
 set +e
-grep -Fxv -f "$w/cacert.fp" "$w/certifi.fp" > "$w/diff.fp"
+grep -Fxnv -f "$w/cacert.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/diff.ln"
 set -e
 
-if [ -s "$w/diff.fp" ] ; then
+if [ -s "$w/diff.ln" ] ; then
-	set +e
+	terse_fingerprint() { cut -d = -f 2- | tr -cd '[:alnum:]' ; }
-	grep -Fxn -f "$w/diff.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/records.diff"
-	set -e
-
-	terse_fingerprint() {
-		cut -d = -f 2- | tr -cd '[:alnum:]'
-	}
-
-	mkdir "$w/extras"
 
 	while read -r n ; do
 		[ -n "$n" ] || continue
 
 		fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint)
 		off=$(sed -ne "${n}p" "$w/certifi.off")
-		sed -ne "${off}p" "$w/certifi.pem" | openssl x509 > "${dst_dir}/certifi-${fp}.crt"
+		sed -ne "${off}p" "$w/certifi.pem" > "${dst_dir}/certifi-${fp}.crt"
-	done < "$w/records.diff"
+	done < "$w/diff.ln"
 fi
 
 rm -rf "$w" ; unset w

image-entry.d/77-openssl-ca-certs.envsh

@@ -30,37 +30,43 @@ while : ; do
 	unset orig_ca_file
 	[ -s "$w/all.pem" ] || break
 
-	openssl-cert-fingerprint.sh "$w/all.pem" | sort -uV > "$w/all.fp"
+	openssl-cert-auto-pem.sh "$w/all.pem" "$w/new.pem" "$w/new.fp" "$w/new.off"
-	[ -s "$w/all.fp" ] || break
+	[ -s "$w/new.pem" ] || break
+	[ -s "$w/new.fp" ] || break
+	[ -s "$w/new.off" ] || break
+	rm -f "$w/all.pem"
 
 	## leaving processing section
 	rm -f "$w/processing"
 
-	unset dev_root dev_bundle dev_bundle_fp
-	dev_root=$(env stat -c '%d' / )
-	dev_bundle=$(env stat -L -c '%d' "${def_bundle}")
-	dev_bundle_fp=$(env stat -L -c '%d' "${def_bundle_fp}")
-
 	unset def_bundle_bind_mount
 	def_bundle_bind_mount=1
 	while : ; do
-		[ "${dev_root}"   = "${dev_bundle}" ]    || break
+		unset devno_root devno_bundle devno_bundle_fp
-		[ "${dev_root}"   = "${dev_bundle_fp}" ] || break
+		devno_root=$(env stat -c '%d' / )
-		[ "${dev_bundle}" = "${dev_bundle_fp}" ] || break
+
+		[ -f "${def_bundle}" ] || break
+		devno_bundle=$(env stat -L -c '%d' "${def_bundle}")
+		[ "${devno_root}" = "${devno_bundle}" ] || break
+
+		[ -f "${def_bundle_fp}" ] || break
+		devno_bundle_fp=$(env stat -L -c '%d' "${def_bundle_fp}")
+		[ "${devno_root}" = "${devno_bundle_fp}" ] || break
 
 		def_bundle_bind_mount=0
 	break ; done
-	unset dev_root dev_bundle dev_bundle_fp
+	unset devno_root devno_bundle devno_bundle_fp
 
 	if [ "${def_bundle_bind_mount}" = 1 ] ; then
 		log_always "detected bind-mount inside ${def_bundle%/*}/"
 		log_always "this is merely misuse!"
 
-		openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem"
+		if [ -s "${def_bundle}" ] ; then
-		openssl-cert-fingerprint.sh "$w/cacert.pem" | sort -uV > "$w/cacert.fp"
+			openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp"
+		fi
 	else
-		ln -s "${def_bundle}"    "$w/cacert.pem"
+		ln -s "${def_bundle}"     "$w/cacert.pem"
-		ln -s "${def_bundle_fp}" "$w/cacert.fp"
+		ln -s "${def_bundle_fp}"  "$w/cacert.fp"
 	fi
 
 	unset with_def_bundle
@@ -73,19 +79,10 @@ while : ; do
 	break ; done
 
 	if [ "${with_def_bundle}" = 1 ] ; then
-		grep -Fxv -f "$w/cacert.fp" "$w/all.fp" > "$w/diff.fp"
+		grep -Fxnv -f "$w/cacert.fp" "$w/new.fp" | cut -d : -f 1 > "$w/diff.ln"
-		[ -s "$w/diff.fp" ] || break
+		[ -s "$w/diff.ln" ] || break
-
-		## entering processing section
-		touch "$w/processing"
-
-		grep -Fxn -f "$w/diff.fp" "$w/all.fp" | cut -d : -f 1 > "$w/diff.lineno"
-		[ -s "$w/diff.lineno" ] || break
-
-		## leaving processing section
-		rm -f "$w/processing"
 	else
-		: > "$w/diff.lineno"
+		: > "$w/diff.ln"
 	fi
 
 	: > "${volume_root}/ca.pem"
@@ -99,9 +96,11 @@ while : ; do
 	while read -r n ; do
 		[ -n "$n" ] || continue
 
-		off=$(sed -ne "${n}p" "$w/all.off")
+		off=$(sed -ne "${n}p" "$w/new.off")
-		sed -ne "${off}p" "$w/all.pem" | openssl x509
+		[ -n "${off}" ] || continue
-	done < "$w/diff.lineno" >> "${volume_root}/ca.pem"
+
+		sed -ne "${off}p" "$w/new.pem"
+	done < "$w/diff.ln" >> "${volume_root}/ca.pem"
 	unset n off
 
 	set -a
@@ -110,10 +109,14 @@ while : ; do
 	SSL_CERT_DIR="${empty_dir}"
 	set +a
 break ; done
-unset def_bundle_bind_mount with_def_bundle
+unset def_bundle_fp def_bundle_bind_mount with_def_bundle
 
-[ -f "${volume_root}/ca.pem" ] || ln -s "${def_bundle}" "${volume_root}/ca.pem"
+while ! [ -f "${volume_root}/ca.pem" ] ; do
-unset def_bundle def_bundle_fp
+	[ -s "${def_bundle}" ] || break
+	ln -s "${def_bundle}" "${volume_root}/ca.pem"
+break ; done
+unset def_bundle
+[ -f "${volume_root}/ca.pem" ] || : > "${volume_root}/ca.pem"
 
 if [ -n "${w:-}" ] ; then
 	if [ -f "$w/processing" ] ; then

requirements.txt

@@ -0,0 +1,5 @@
+jinja2==3.1.4
+netaddr==1.3.0
+psutil==6.0.0
+pyyaml==6.0.2
+wcmatch==9.0

scripts/openssl-cert-auto-pem.sh

@@ -13,38 +13,83 @@ me=${0##*/}
 
 w=$(mktemp -d) || exit 1
 w_cleanup() {
-	[ -z "$w" ] || ls -lA "$w/"
+	[ -z "$w" ] || ls -lA "$w/" >&2
 	[ -z "$w" ] || rm -rf "$w"
 	unset w
 	exit "${1:-0}"
 }
 
+bundle_offsets() {
+	awk '
+	BEGIN { OFS = "," ; i_begin = 0 ; }
+	$0 == "-----BEGIN CERTIFICATE-----" { i_begin = NR ; }
+	$0 == "-----END CERTIFICATE-----"   { if (i_begin > 0) { print i_begin,NR ; i_begin = 0 ; } }
+	' "$1"
+}
+
+bundle_fingerprints() {
+	local x f
+	while read -r x ; do
+		[ -n "$x" ] || continue
+
+		f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint -sha256)
+		[ -n "$f" ] || f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint)
+		[ -n "$f" ] || continue
+
+		printf '%s\n' "$f" | tr '[:upper:]' '[:lower:]'
+	done < "$2"
+}
+
 openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1
 [ -s "$w/cert.pem" ] || w_cleanup 1
 tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt"
 [ -s "$w/cert.txt" ] || w_cleanup 1
+rm -f "$w/cert.pem"
 
-awk '
+bundle_offsets "$w/cert.txt" > "$w/cert.off"
-BEGIN {
+[ -s "$w/cert.off" ] || w_cleanup 1
-    OFS = ","
-    m_begin="-----BEGIN CERTIFICATE-----"
-    m_end="-----END CERTIFICATE-----"
-    i_begin = 0
-}
-$0 == m_begin { i_begin = NR ; }
-$0 == m_end {
-    if (i_begin > 0) {
-        print i_begin,NR
-        i_begin = 0
-    }
-}
-' "$w/cert.txt" > "$w/cert.offsets"
-[ -s "$w/cert.offsets" ] || w_cleanup 1
 
-while read -r a ; do
+bundle_fingerprints "$w/cert.txt" "$w/cert.off" > "$w/cert.fp.all"
-	[ -n "$a" ] || continue
+[ -s "$w/cert.fp.all" ] || w_cleanup 1
 
-	sed -ne "${a}p" "$w/cert.txt"
+sort -uV < "$w/cert.fp.all" > "$w/cert.fp"
-done < "$w/cert.offsets"
+while read -r fp ; do
+	[ -n "${fp}" ] || continue
+
+	n=$(grep -m1 -Fxn -e "${fp}" "$w/cert.fp.all" | cut -d : -f 1)
+	[ -n "$n" ] || continue
+
+	off=$(sed -ne "${n}p" "$w/cert.off")
+	[ -n "${off}" ] || continue
+
+	sed -ne "${off}p" "$w/cert.txt"
+done < "$w/cert.fp" > "$w/cert.pem"
+[ -s "$w/cert.pem" ] || w_cleanup 1
+rm -f "$w/cert.txt" "$w/cert.off" "$w/cert.fp.all"
+
+if [ -n "$2" ] ; then
+	while : ; do
+		if [ -e "$2" ] ; then
+			[ -f "$2" ] || break
+		fi
+		cat > "$2"
+	break ; done
+else
+	cat
+fi < "$w/cert.pem"
+
+while [ -n "$3" ] ; do
+	if [ -e "$3" ] ; then
+		[ -f "$3" ] || break
+	fi
+	cat "$w/cert.fp" > "$3"
+break ; done
+
+while [ -n "$4" ] ; do
+	if [ -e "$4" ] ; then
+		[ -f "$4" ] || break
+	fi
+	bundle_offsets "$w/cert.pem" > "$4"
+break ; done
 
 rm -rf "$w" ; unset w

scripts/openssl-cert-fingerprint.sh

@@ -1,52 +0,0 @@
-#!/bin/sh
-set -f
-
-[ $# -gt 0 ] || exit 0
-me=${0##*/}
-
-[ -n "$1" ] || exit 1
-[ -f "$1" ] || {
-	env printf '%s: not a file or does not exist: %q\n' "${me}" "$1" >&2
-	exit 1
-}
-[ -s "$1" ] || exit 0
-
-w=$(mktemp -d) || exit 1
-w_cleanup() {
-	[ -z "$w" ] || ls -lA "$w/"
-	[ -z "$w" ] || rm -rf "$w"
-	unset w
-	exit "${1:-0}"
-}
-
-openssl-cert-auto-pem.sh "$1" > "$w/cert.pem" || w_cleanup 1
-[ -s "$w/cert.pem" ] || w_cleanup 1
-
-awk '
-BEGIN {
-    OFS = ","
-    m_begin="-----BEGIN CERTIFICATE-----"
-    m_end="-----END CERTIFICATE-----"
-    i_begin = 0
-}
-$0 == m_begin { i_begin = NR ; }
-$0 == m_end {
-    if (i_begin > 0) {
-        print i_begin,NR
-        i_begin = 0
-    }
-}
-' "$w/cert.pem" > "$w/cert.off"
-[ -s "$w/cert.off" ] || w_cleanup 1
-
-while read -r a ; do
-	[ -n "$a" ] || continue
-
-	{
-		sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint -sha256 \
-		|| \
-		sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint
-	} | tr '[:upper:]' '[:lower:]'
-done < "$w/cert.off"
-
-w_cleanup 0