1
0

treewide: rework certificate handling

also:
- reorder some blocks in dockerfiles
- provide sane requirements.txt
This commit is contained in:
Konstantin Demin 2024-09-20 02:39:40 +03:00
parent 62eeabff2b
commit 91ec2a4c86
Signed by: krd
GPG Key ID: 4D56F87A8BA65FD0
8 changed files with 174 additions and 203 deletions

View File

@ -3,32 +3,31 @@ FROM docker.io/rockdrilla/angie-conv:${IMAGE_VERSION}-deps AS deps
## ---
FROM deps AS certs
FROM deps AS setup
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
## consult https://github.com/certifi/python-certifi/
ENV CERTIFI_COMMIT=bd8153872e9c6fc98f4023df9c2deaffea2fa463
ADD https://angie.software/keys/angie-signing.gpg /tmp/angie.gpg.bin
COPY /apt/sources.angie /etc/apt/sources.list.d/angie.txt
RUN apt-install.sh ca-certificates ; \
## process certifi
ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp.orig" ; \
ls -l "${ca_file}" ; \
certifi-extras.sh ; \
openssl-cert-fingerprint.sh "${ca_file}" | sort -uV > "${ca_file}.fp" ; \
chmod 0444 "${ca_file}" "${ca_file}.fp" "${ca_file}.fp.orig" ; \
ls -l "${ca_file}" "${ca_file}.fp" "${ca_file}.fp.orig"
RUN pkg='gnupg' ; \
apt-install.sh ${pkg} ; \
## process Angie GPG keyring / APT sources
gpg-export.sh /tmp/angie.gpg.bin /etc/apt/keyrings/angie.gpg.asc ; \
rm -f /tmp/angie.gpg.bin ; \
env -C /etc/apt/sources.list.d mv angie.txt angie.sources ; \
## verify sources!
apt-env.sh apt-get update ; \
apt-remove.sh ${pkg} ; \
apt-clean.sh
## ---
FROM deps AS pycache
SHELL [ "/bin/sh", "-ec" ]
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
@ -74,7 +73,8 @@ SHELL [ "/bin/sh", "-ec" ]
COPY /Dockerfile /usr/local/share/
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
COPY --from=setup /etc/apt/keyrings/angie.gpg.asc /etc/apt/keyrings/
COPY --from=setup /etc/apt/sources.list.d/angie.sources /etc/apt/sources.list.d/
## RFC: Python cache
## TODO: reduce load by selecting only __pycache__ directories in either way

View File

@ -2,7 +2,7 @@
ARG PYTHONTAG=3.11.10-slim-bookworm
FROM docker.io/python:${PYTHONTAG} AS base-upstream
FROM base-upstream AS base
FROM base-upstream AS base-intermediate
SHELL [ "/bin/sh", "-ec" ]
COPY /Dockerfile.base /usr/local/share/
@ -223,5 +223,39 @@ RUN find /usr/local/sbin/ ! -type d -ls -delete ; \
find /run/ -mindepth 1 -ls -delete || : ; \
install -d -m 01777 /run/lock
## ---
FROM base-intermediate AS certs
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
## "2024.08.30"
ENV CERTIFI_COMMIT=325c2fde4f8eec10d682b09f3b0414dc05e69a81
# 'https://raw.githubusercontent.com/certifi/python-certifi'
ARG CERTIFI_BASE_URI='https://github.com/certifi/python-certifi/raw'
ARG CERTIFI_URI="${CERTIFI_BASE_URI}/${CERTIFI_COMMIT}/certifi/cacert.pem"
ADD "${CERTIFI_URI}" /tmp/certifi.crt
RUN apt-install.sh ca-certificates ; \
apt-clean.sh ; \
ca_file='/etc/ssl/certs/ca-certificates.crt' ; \
ls -l "${ca_file}" ; \
## process certifi
certifi-extras.sh /tmp/certifi.crt ; \
openssl-cert-auto-pem.sh "${ca_file}" "${ca_file}.new" "${ca_file}.fp" ; \
mv -f "${ca_file}.new" "${ca_file}" ; \
chmod 0444 "${ca_file}" "${ca_file}.fp" ; \
ls -l "${ca_file}" "${ca_file}.fp"
## ---
FROM base-intermediate AS base
COPY --from=certs /etc/ssl/certs/ca-certificates.* /etc/ssl/certs/
ENTRYPOINT [ ]
CMD [ "bash" ]

View File

@ -3,27 +3,14 @@ FROM docker.io/rockdrilla/angie-conv:${IMAGE_VERSION}-base AS base
## ---
FROM base AS setup
FROM base AS build
SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
ADD https://angie.software/keys/angie-signing.gpg /tmp/angie.gpg.bin
COPY /apt/sources.angie /etc/apt/sources.list.d/angie.txt
COPY /requirements.txt /tmp/
RUN pkg='gnupg' ; \
apt-install.sh ${pkg} ; \
## process Angie GPG keyring / APT sources
gpg-export.sh /tmp/angie.gpg.bin /etc/apt/keyrings/angie.gpg.asc ; \
rm -f /tmp/angie.gpg.bin ; \
env -C /etc/apt/sources.list.d mv angie.txt angie.sources ; \
## verify sources!
apt-env.sh apt-get update ; \
apt-remove.sh ${pkg} ; \
apt-clean.sh
ENV INSTALL_WHEELS='jinja2 netaddr psutil pyyaml wcmatch'
ENV DEV_PACKAGES='libyaml-dev'
# markupsafe, psutil
ENV CIBUILDWHEEL=1
@ -43,7 +30,7 @@ RUN w=$(mktemp -d) ; : "${w:?}" ; \
rm -rf "$w/" ; unset w ; \
apt-install.sh build-essential ; \
pip-env.sh pip install 'cython' ; \
pip-env.sh pip install --no-binary :all: ${INSTALL_WHEELS} ; \
pip-env.sh pip install --no-binary :all: -r /tmp/requirements.txt ; \
pip-env.sh pip uninstall -y 'cython' ; \
python-rm-cache.sh "${PYTHON_SITE_PACKAGES}" ; \
rm -rf \
@ -67,12 +54,9 @@ SHELL [ "/bin/sh", "-ec" ]
COPY /Dockerfile.deps /usr/local/share/
COPY --from=setup /etc/apt/keyrings/angie.gpg.asc /etc/apt/keyrings/
COPY --from=setup /etc/apt/sources.list.d/angie.sources /etc/apt/sources.list.d/
## Python: site-packages
COPY --from=setup /usr/local/bin/ /usr/local/bin/
COPY --from=setup /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/
COPY --from=build /usr/local/bin/ /usr/local/bin/
COPY --from=build /${PYTHON_SITE_PACKAGES}/ /${PYTHON_SITE_PACKAGES}/
COPY /scripts/* /usr/local/sbin/

View File

@ -1,89 +1,41 @@
#!/bin/sh
set -ef
certifi_uri="https://raw.githubusercontent.com/certifi/python-certifi/${CERTIFI_COMMIT:?}/certifi/cacert.pem"
dst_dir=/usr/local/share/ca-certificates
w=$(mktemp -d) ; : "${w:?}"
w_cleanup() {
[ -z "$w" ] || ls -lA "$w/"
[ -z "$w" ] || ls -lA "$w/" >&2
[ -z "$w" ] || rm -rf "$w"
unset w
exit "${1:-0}"
}
curl -sSL "${certifi_uri}" > "$w/certifi.crt"
def_bundle='/etc/ssl/certs/ca-certificates.crt'
openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem"
openssl-cert-auto-pem.sh "$w/certifi.crt" > "$w/certifi.pem"
openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp"
[ -s "$w/cacert.pem" ] || w_cleanup 1
[ -s "$w/cacert.fp" ] || w_cleanup 1
openssl-cert-auto-pem.sh "$1" "$w/certifi.pem" "$w/certifi.fp" "$w/certifi.off"
[ -s "$w/certifi.pem" ] || w_cleanup 1
bundle_offsets() {
awk '
BEGIN {
OFS = ","
m_begin="-----BEGIN CERTIFICATE-----"
m_end="-----END CERTIFICATE-----"
i_begin = 0
}
$0 == m_begin { i_begin = NR ; }
$0 == m_end {
if (i_begin > 0) {
print i_begin,NR
i_begin = 0
}
}
' "$1"
}
bundle_offsets "$w/cacert.pem" > "$w/cacert.off"
bundle_offsets "$w/certifi.pem" > "$w/certifi.off"
[ -s "$w/cacert.off" ] || w_cleanup 1
[ -s "$w/certifi.fp" ] || w_cleanup 1
[ -s "$w/certifi.off" ] || w_cleanup 1
bundle_fingerprints() {
local a
while read -r a ; do
[ -n "$a" ] || continue
{
sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint -sha256 \
|| \
sed -ne "${a}p" "$1" | openssl x509 -noout -fingerprint
} | tr '[:upper:]' '[:lower:]'
done < "$2"
}
bundle_fingerprints "$w/cacert.pem" "$w/cacert.off" | sort -uV > "$w/cacert.fp"
bundle_fingerprints "$w/certifi.pem" "$w/certifi.off" | sort -uV > "$w/certifi.fp"
[ -s "$w/cacert.fp" ] || w_cleanup 1
[ -s "$w/certifi.fp" ] || w_cleanup 1
set +e
grep -Fxv -f "$w/cacert.fp" "$w/certifi.fp" > "$w/diff.fp"
grep -Fxnv -f "$w/cacert.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/diff.ln"
set -e
if [ -s "$w/diff.fp" ] ; then
set +e
grep -Fxn -f "$w/diff.fp" "$w/certifi.fp" | cut -d : -f 1 > "$w/records.diff"
set -e
terse_fingerprint() {
cut -d = -f 2- | tr -cd '[:alnum:]'
}
mkdir "$w/extras"
if [ -s "$w/diff.ln" ] ; then
terse_fingerprint() { cut -d = -f 2- | tr -cd '[:alnum:]' ; }
while read -r n ; do
[ -n "$n" ] || continue
fp=$(sed -ne "${n}p" "$w/certifi.fp" | terse_fingerprint)
off=$(sed -ne "${n}p" "$w/certifi.off")
sed -ne "${off}p" "$w/certifi.pem" | openssl x509 > "${dst_dir}/certifi-${fp}.crt"
done < "$w/records.diff"
sed -ne "${off}p" "$w/certifi.pem" > "${dst_dir}/certifi-${fp}.crt"
done < "$w/diff.ln"
fi
rm -rf "$w" ; unset w

View File

@ -30,34 +30,40 @@ while : ; do
unset orig_ca_file
[ -s "$w/all.pem" ] || break
openssl-cert-fingerprint.sh "$w/all.pem" | sort -uV > "$w/all.fp"
[ -s "$w/all.fp" ] || break
openssl-cert-auto-pem.sh "$w/all.pem" "$w/new.pem" "$w/new.fp" "$w/new.off"
[ -s "$w/new.pem" ] || break
[ -s "$w/new.fp" ] || break
[ -s "$w/new.off" ] || break
rm -f "$w/all.pem"
## leaving processing section
rm -f "$w/processing"
unset dev_root dev_bundle dev_bundle_fp
dev_root=$(env stat -c '%d' / )
dev_bundle=$(env stat -L -c '%d' "${def_bundle}")
dev_bundle_fp=$(env stat -L -c '%d' "${def_bundle_fp}")
unset def_bundle_bind_mount
def_bundle_bind_mount=1
while : ; do
[ "${dev_root}" = "${dev_bundle}" ] || break
[ "${dev_root}" = "${dev_bundle_fp}" ] || break
[ "${dev_bundle}" = "${dev_bundle_fp}" ] || break
unset devno_root devno_bundle devno_bundle_fp
devno_root=$(env stat -c '%d' / )
[ -f "${def_bundle}" ] || break
devno_bundle=$(env stat -L -c '%d' "${def_bundle}")
[ "${devno_root}" = "${devno_bundle}" ] || break
[ -f "${def_bundle_fp}" ] || break
devno_bundle_fp=$(env stat -L -c '%d' "${def_bundle_fp}")
[ "${devno_root}" = "${devno_bundle_fp}" ] || break
def_bundle_bind_mount=0
break ; done
unset dev_root dev_bundle dev_bundle_fp
unset devno_root devno_bundle devno_bundle_fp
if [ "${def_bundle_bind_mount}" = 1 ] ; then
log_always "detected bind-mount inside ${def_bundle%/*}/"
log_always "this is merely misuse!"
openssl-cert-auto-pem.sh "${def_bundle}" > "$w/cacert.pem"
openssl-cert-fingerprint.sh "$w/cacert.pem" | sort -uV > "$w/cacert.fp"
if [ -s "${def_bundle}" ] ; then
openssl-cert-auto-pem.sh "${def_bundle}" "$w/cacert.pem" "$w/cacert.fp"
fi
else
ln -s "${def_bundle}" "$w/cacert.pem"
ln -s "${def_bundle_fp}" "$w/cacert.fp"
@ -73,19 +79,10 @@ while : ; do
break ; done
if [ "${with_def_bundle}" = 1 ] ; then
grep -Fxv -f "$w/cacert.fp" "$w/all.fp" > "$w/diff.fp"
[ -s "$w/diff.fp" ] || break
## entering processing section
touch "$w/processing"
grep -Fxn -f "$w/diff.fp" "$w/all.fp" | cut -d : -f 1 > "$w/diff.lineno"
[ -s "$w/diff.lineno" ] || break
## leaving processing section
rm -f "$w/processing"
grep -Fxnv -f "$w/cacert.fp" "$w/new.fp" | cut -d : -f 1 > "$w/diff.ln"
[ -s "$w/diff.ln" ] || break
else
: > "$w/diff.lineno"
: > "$w/diff.ln"
fi
: > "${volume_root}/ca.pem"
@ -99,9 +96,11 @@ while : ; do
while read -r n ; do
[ -n "$n" ] || continue
off=$(sed -ne "${n}p" "$w/all.off")
sed -ne "${off}p" "$w/all.pem" | openssl x509
done < "$w/diff.lineno" >> "${volume_root}/ca.pem"
off=$(sed -ne "${n}p" "$w/new.off")
[ -n "${off}" ] || continue
sed -ne "${off}p" "$w/new.pem"
done < "$w/diff.ln" >> "${volume_root}/ca.pem"
unset n off
set -a
@ -110,10 +109,14 @@ while : ; do
SSL_CERT_DIR="${empty_dir}"
set +a
break ; done
unset def_bundle_bind_mount with_def_bundle
unset def_bundle_fp def_bundle_bind_mount with_def_bundle
[ -f "${volume_root}/ca.pem" ] || ln -s "${def_bundle}" "${volume_root}/ca.pem"
unset def_bundle def_bundle_fp
while ! [ -f "${volume_root}/ca.pem" ] ; do
[ -s "${def_bundle}" ] || break
ln -s "${def_bundle}" "${volume_root}/ca.pem"
break ; done
unset def_bundle
[ -f "${volume_root}/ca.pem" ] || : > "${volume_root}/ca.pem"
if [ -n "${w:-}" ] ; then
if [ -f "$w/processing" ] ; then

5
requirements.txt Normal file
View File

@ -0,0 +1,5 @@
jinja2==3.1.4
netaddr==1.3.0
psutil==6.0.0
pyyaml==6.0.2
wcmatch==9.0

View File

@ -13,38 +13,83 @@ me=${0##*/}
w=$(mktemp -d) || exit 1
w_cleanup() {
[ -z "$w" ] || ls -lA "$w/"
[ -z "$w" ] || ls -lA "$w/" >&2
[ -z "$w" ] || rm -rf "$w"
unset w
exit "${1:-0}"
}
bundle_offsets() {
awk '
BEGIN { OFS = "," ; i_begin = 0 ; }
$0 == "-----BEGIN CERTIFICATE-----" { i_begin = NR ; }
$0 == "-----END CERTIFICATE-----" { if (i_begin > 0) { print i_begin,NR ; i_begin = 0 ; } }
' "$1"
}
bundle_fingerprints() {
local x f
while read -r x ; do
[ -n "$x" ] || continue
f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint -sha256)
[ -n "$f" ] || f=$(sed -ne "${x}p" "$1" | openssl x509 -noout -fingerprint)
[ -n "$f" ] || continue
printf '%s\n' "$f" | tr '[:upper:]' '[:lower:]'
done < "$2"
}
openssl storeutl -certs "$1" > "$w/cert.pem" || w_cleanup 1
[ -s "$w/cert.pem" ] || w_cleanup 1
tr -s '\r\n' '\n' < "$w/cert.pem" > "$w/cert.txt"
[ -s "$w/cert.txt" ] || w_cleanup 1
rm -f "$w/cert.pem"
awk '
BEGIN {
OFS = ","
m_begin="-----BEGIN CERTIFICATE-----"
m_end="-----END CERTIFICATE-----"
i_begin = 0
}
$0 == m_begin { i_begin = NR ; }
$0 == m_end {
if (i_begin > 0) {
print i_begin,NR
i_begin = 0
}
}
' "$w/cert.txt" > "$w/cert.offsets"
[ -s "$w/cert.offsets" ] || w_cleanup 1
bundle_offsets "$w/cert.txt" > "$w/cert.off"
[ -s "$w/cert.off" ] || w_cleanup 1
while read -r a ; do
[ -n "$a" ] || continue
bundle_fingerprints "$w/cert.txt" "$w/cert.off" > "$w/cert.fp.all"
[ -s "$w/cert.fp.all" ] || w_cleanup 1
sed -ne "${a}p" "$w/cert.txt"
done < "$w/cert.offsets"
sort -uV < "$w/cert.fp.all" > "$w/cert.fp"
while read -r fp ; do
[ -n "${fp}" ] || continue
n=$(grep -m1 -Fxn -e "${fp}" "$w/cert.fp.all" | cut -d : -f 1)
[ -n "$n" ] || continue
off=$(sed -ne "${n}p" "$w/cert.off")
[ -n "${off}" ] || continue
sed -ne "${off}p" "$w/cert.txt"
done < "$w/cert.fp" > "$w/cert.pem"
[ -s "$w/cert.pem" ] || w_cleanup 1
rm -f "$w/cert.txt" "$w/cert.off" "$w/cert.fp.all"
if [ -n "$2" ] ; then
while : ; do
if [ -e "$2" ] ; then
[ -f "$2" ] || break
fi
cat > "$2"
break ; done
else
cat
fi < "$w/cert.pem"
while [ -n "$3" ] ; do
if [ -e "$3" ] ; then
[ -f "$3" ] || break
fi
cat "$w/cert.fp" > "$3"
break ; done
while [ -n "$4" ] ; do
if [ -e "$4" ] ; then
[ -f "$4" ] || break
fi
bundle_offsets "$w/cert.pem" > "$4"
break ; done
rm -rf "$w" ; unset w

View File

@ -1,52 +0,0 @@
#!/bin/sh
set -f
[ $# -gt 0 ] || exit 0
me=${0##*/}
[ -n "$1" ] || exit 1
[ -f "$1" ] || {
env printf '%s: not a file or does not exist: %q\n' "${me}" "$1" >&2
exit 1
}
[ -s "$1" ] || exit 0
w=$(mktemp -d) || exit 1
w_cleanup() {
[ -z "$w" ] || ls -lA "$w/"
[ -z "$w" ] || rm -rf "$w"
unset w
exit "${1:-0}"
}
openssl-cert-auto-pem.sh "$1" > "$w/cert.pem" || w_cleanup 1
[ -s "$w/cert.pem" ] || w_cleanup 1
awk '
BEGIN {
OFS = ","
m_begin="-----BEGIN CERTIFICATE-----"
m_end="-----END CERTIFICATE-----"
i_begin = 0
}
$0 == m_begin { i_begin = NR ; }
$0 == m_end {
if (i_begin > 0) {
print i_begin,NR
i_begin = 0
}
}
' "$w/cert.pem" > "$w/cert.off"
[ -s "$w/cert.off" ] || w_cleanup 1
while read -r a ; do
[ -n "$a" ] || continue
{
sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint -sha256 \
|| \
sed -ne "${a}p" "$w/cert.pem" | openssl x509 -noout -fingerprint
} | tr '[:upper:]' '[:lower:]'
done < "$w/cert.off"
w_cleanup 0