krd/angie-conv-image
1
0
Fork 0
Code

refactor environment handling

Browse Source 
also:
- drop "njs" directories (not actually used nor useful)
- rework jinja filters: more functions, shorter names, etc.
This commit is contained in:
Konstantin Demin 2024-07-23 00:02:19 +03:00
parent 1dda7066c9
commit a0a4531501
Signed by: krd
GPG Key ID: 4D56F87A8BA65FD0
22 changed files with 487 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -37,7 +37,7 @@ RUN python3 -m compileall -q -j 2 /usr/local/lib/j2cfg/
RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
| sed -zEn \
-e '/^(asyncio|collections|concurrent|encodings|html|importlib|json|logging|multiprocessing|re|urllib|xml)$/p' \
-e '/^(collections|importlib|json|re)$/p' \
| sort -zV \
| env -C "${libpython}" xargs -0r \
python3 -m compileall -q -j 2 ; \
@ -141,7 +141,7 @@ RUN install -d -o angie -g angie -m 03777 /angie /run/angie ; \
ln -sv /run/angie/lock lock.d ; \
ln -sv ${ANGIE_MODULES_DIR} modules.dist ; \
## hyper-modular paths:
data='conf j2cfg mod modules njs site snip static' ; \
data='conf j2cfg mod modules site snip static' ; \
vardata='cache lib log' ; \
for n in ${data} ; do \
for d in "$n" "$n.dist" ; do \

19
angie/conf.dist/core-preserve-env.conf.j2
View File

@ -1,19 +0,0 @@
{#- prologue -#}
{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%}
{%- set have_tz = preserve_env|is_str_list_re_match('TZ(=|$)') -%}
{%- set have_malloc_arena = preserve_env|is_str_list_re_match('MALLOC_ARENA_MAX(=|$)') -%}
{#- main part -#}
{%- if not have_tz -%}
env TZ;
{% endif %}
{%- if not have_malloc_arena -%}
env MALLOC_ARENA_MAX;
{% endif %}
{%- for v in preserve_env -%}
{%- if re.search("(\"|'|\\s)", v) %}
{#- TODO: investigate corrent escape behavior for Angie/nginx -#}
env {{ v.__repr__() }};
{%- else %}
env {{ v }};
{%- endif %}
{%- endfor -%}

28
angie/conf.dist/core-worker-env.conf.j2 Normal file
View File

@ -0,0 +1,28 @@
{#- prologue -#}
{#- NB: "TZ" is always provided by Angie itself -#}
{%- set s_vars = ['MALLOC_ARENA_MAX', 'GLIBC_TUNABLES', 'MALLOC_CONF'] -%}
{%- set c_env = ( j2cfg.core_worker_env or [] ) | any_to_env_dict -%}
{%- set c_vars = c_env | dict_keys -%}
{%- set c_vars_preserve = c_env | dict_empty_keys -%}
{%- set c_vars_override = c_env | dict_non_empty_keys -%}
{%- set vars_preserve = ( c_vars_preserve + ( s_vars | list_diff(c_vars) )) | sort -%}
{#- main part -#}
{%- if vars_preserve %}
## preserve
{%- for k in vars_preserve %}
env {{ k }};
{%- endfor %}
{% endif %}
{%- if c_vars_override %}
## WARNING!
## explicit environment variables are NOT implemented
## reason: envs are supported only for http_perl but not for http_js/stream_js
## solution: provide environment variables explicitly
## and then list them in "core_worker_env" key in config
{%- for k in c_vars_override %}
{%- set v = c_env[k] -%}
# env {{ k }}={{ v.__repr__() }};
{%- endif %}
{%- endfor %}
{% endif %}

7
angie/j2cfg.dist/core-preserve-environment.txt.j2
View File

@ -1,7 +0,0 @@
{#- prologue -#}
{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%}
{%- set preserve_vars = preserve_env|str_list_re_fullmatch('[^=]+') -%}
{#- main part -#}
{% for v in preserve_vars -%}
{{ v }}
{% endfor -%}

10
angie/j2cfg.dist/core-worker-env.txt.j2 Normal file
View File

@ -0,0 +1,10 @@
{#- prologue -#}
{%- set s_vars = ['MALLOC_ARENA_MAX', 'GLIBC_TUNABLES', 'MALLOC_CONF'] -%}
{%- set c_env = ( j2cfg.core_worker_env or [] ) | any_to_env_dict -%}
{%- set c_vars = c_env | dict_keys -%}
{%- set c_vars_preserve = c_env | dict_empty_keys -%}
{%- set vars_preserve = ( c_vars_preserve + ( s_vars | list_diff(c_vars) )) | sort -%}
{#- main part -#}
{%- for k in vars_preserve -%}
{{ k }}
{% endfor -%}

60
image-entry.d/00-common.envsh
View File

@ -12,7 +12,7 @@ have_envvar() {
## unexporting variable in (POSIX) sh is PITA =/
unexport() {
unset ___k ___v
local ___k ___v
for ___k ; do
[ -n "${___k}" ] || continue
have_envvar "${___k}" || continue
@ -20,9 +20,7 @@ unexport() {
___v=$(eval printf '%s' "\"\${${___k}}\"")
eval "unset ${___k}"
eval "${___k}=$(env printf '%s' \"\${___v}\")"
unset ___v
done
unset ___k
}
## likely the same as in https://pkg.go.dev/strconv#ParseBool
@ -38,31 +36,33 @@ gobool_to_int() {
[ -n "${__IEP_SRC:-}" ] || __IEP_SRC="$0"
IEP_TRACE=$(gobool_to_int "${IEP_TRACE:-0}" 0)
export IEP_TRACE
log_always() {
if [ "${IEP_TRACE}" = 1 ] ; then
if [ "${IEP_DEBUG}" = 1 ] ; then
echo "# $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): ${__IEP_SRC}${*:+: $*}"
else
echo "# ${__IEP_SRC}${*:+: $*}"
fi >&2
}
IEP_VERBOSE=$(gobool_to_int "${IEP_VERBOSE:-${IEP_TRACE}}" "${IEP_TRACE}")
export IEP_VERBOSE
log() {
[ "${IEP_VERBOSE}" = 0 ] || log_always "$@"
}
log_file() { sed -E '/^./s,^, ,' < "$1" >&2 ; }
ln_s() {
if [ "${IEP_VERBOSE}" = 0 ] ; then
ln_s() { ln -s "$@" || return; }
cp_a() { cp -a "$@" || return; }
ln -s "$@" || return
else
ln_s() { ln -sv "$@" || return; }
cp_a() { cp -av "$@" || return; }
ln -sv "$@" || return
fi
}
cp_a() {
if [ "${IEP_VERBOSE}" = 0 ] ; then
cp -a "$@" || return
else
cp -av "$@" || return
fi
}
ln_cp() {
if [ -h "$1" ] ; then
@ -82,7 +82,7 @@ untemplate_path() {
"${volume_root}"/* | /etc/angie/run/* )
strip_suffix "$1" "$2"
;;
/etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* )
/etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* )
strip_suffix "$1" "$2"
;;
/etc/angie/static.d/* )
@ -118,12 +118,14 @@ install_userdir() {
}
expand_file_envsubst() {
__r=0
local __ret __src __dst
__ret=0
for __src ; do
[ -n "${__src}" ] || continue
if ! [ -f "${__src}" ] ; then
__r=1
__ret=1
log_always "file not found: ${__src}"
continue
fi
@ -131,24 +133,23 @@ expand_file_envsubst() {
case "${__src}" in
*.in ) ;;
* )
__r=1
__ret=1
log "expand_file_envsubst: file name extension mismatch: ${__src}"
continue
;;
esac
__dest=$(strip_suffix "${__src}" '.in')
if [ -e "${__dest}" ] ; then
__r=1
log "expand_file_envsubst: destination file already exists: ${__dest}"
__dst=$(strip_suffix "${__src}" '.in')
if [ -e "${__dst}" ] ; then
__ret=1
log "expand_file_envsubst: destination file already exists: ${__dst}"
continue
fi
log "Running envsubst: ${__src} -> ${__dest}"
envsubst.sh < "${__src}" > "${__dest}" || __r=1
log "Running envsubst: ${__src} -> ${__dst}"
envsubst.sh < "${__src}" > "${__dst}" || __ret=1
done
unset __src __dest
return ${__r}
return ${__ret}
}
expand_file_j2cfg() {
@ -156,6 +157,8 @@ expand_file_j2cfg() {
}
expand_dir_envsubst() {
local __template_list __have_args __ret __orig_file
__template_list=$(mktemp) || return
find "$@" -follow -type f -name '*.in' \
@ -175,7 +178,6 @@ expand_dir_envsubst() {
[ -n "${__orig_file}" ] || continue
expand_file_envsubst "${__orig_file}" || __ret=1
done < "${__template_list}"
unset __orig_file
if [ -z "${__have_args}" ] ; then
rm -f "${ENVSUBST_ARGS}" ; unset ENVSUBST_ARGS
@ -188,6 +190,8 @@ expand_dir_envsubst() {
}
expand_dir_j2cfg() {
local __template_list __ret
__template_list=$(mktemp) || return
find "$@" -follow -type f -name '*.j2' -printf '%p\0' \
@ -224,10 +228,6 @@ remap_path() {
/etc/angie/modules.dist/* ) echo "${2:-/etc/angie/modules.d}${1#/etc/angie/modules.dist}" ;;
/etc/angie/modules/* ) echo "${2:-/etc/angie/modules.d}${1#/etc/angie/modules}" ;;
/angie/modules/* ) echo "${2:-/etc/angie/modules.d}${1#/angie/modules}" ;;
## njs
/etc/angie/njs.dist/* ) echo "${2:-/etc/angie/njs.d}${1#/etc/angie/njs.dist}" ;;
/etc/angie/njs/* ) echo "${2:-/etc/angie/njs.d}${1#/etc/angie/njs}" ;;
/angie/njs/* ) echo "${2:-/etc/angie/njs.d}${1#/angie/njs}" ;;
## site
/etc/angie/site.dist/* ) echo "${2:-/etc/angie/site.d}${1#/etc/angie/site.dist}" ;;
/etc/angie/site/* ) echo "${2:-/etc/angie/site.d}${1#/etc/angie/site}" ;;

2
image-entry.d/01-defaults.envsh
View File

@ -19,7 +19,7 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then
fi
unset default_dirs_merge default_dirs_link
default_dirs_merge='conf j2cfg mod modules njs site snip'
default_dirs_merge='conf j2cfg mod modules site snip'
default_dirs_link=''
if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then

25
image-entry.d/04-detect-local-ip-addresses.envsh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/sh
## allow these addresses to be provided in case of:
## - local development/testing
## - `hostname -I' random failures or misbehavior
if [ -z "${NGX_IP_ADDRESSES:-}" ] ; then
NGX_IP_ADDRESSES=$(hostname -I)
fi
NGX_IP_ADDRESSES=$(printf '%s' "${NGX_IP_ADDRESSES}" | sed -zE 's/^\s+//;s/\s+$//;s/\s+/ /g')
export NGX_IP_ADDRESSES
unset NGX_IPV4_ADDRESSES NGX_IPV6_ADDRESSES
for i in ${NGX_IP_ADDRESSES} ; do
case "$i" in
*:* )
NGX_IPV6_ADDRESSES="${NGX_IPV6_ADDRESSES:-}${NGX_IPV6_ADDRESSES:+ }$i"
;;
* )
NGX_IPV4_ADDRESSES="${NGX_IPV4_ADDRESSES:-}${NGX_IPV4_ADDRESSES:+ }$i"
;;
esac
done
unset i
export NGX_IPV4_ADDRESSES NGX_IPV6_ADDRESSES

38
image-entry.d/13-core-worker-defaults.envsh
View File

@ -10,9 +10,16 @@ _NGX_WORKER_CONNECTIONS=4096
[ -n "${NGX_WORKER_PROCESSES:-}" ] || NGX_WORKER_PROCESSES=${_NGX_WORKER_PROCESSES}
case "${NGX_WORKER_PROCESSES}" in
"${_NGX_WORKER_PROCESSES}" ) ;;
[1-9] | [1-9][0-9] ) ;;
0 | [Aa][Uu][Tt][Oo] )
log_always "NGX_WORKER_PROCESSES=${NGX_WORKER_PROCESSES} isn't supported yet"
## allow values within [1;999]
[1-9] | [1-9][0-9] | [1-9][0-9][0-9] ) ;;
[Aa][Uu][Tt][Oo] )
## adjust
NGX_WORKER_PROCESSES=auto
log_always "NGX_WORKER_PROCESSES: \"auto\" isn't supported by container yet"
log_always "offloading decision to Angie (this could be a problem!)"
;;
0 )
log_always "NGX_WORKER_PROCESSES: \"0\" isn't supported by container yet"
log_always "setting NGX_WORKER_PROCESSES=${_NGX_WORKER_PROCESSES}"
NGX_WORKER_PROCESSES=${_NGX_WORKER_PROCESSES}
;;
@ -28,6 +35,11 @@ case "${NGX_WORKER_PRIORITY}" in
"${_NGX_WORKER_PRIORITY}" ) ;;
-[1-9] | -1[0-9] | -20 ) ;;
[0-9] | 1[0-9] | 20 ) ;;
-0 )
log_always "NGX_WORKER_PRIORITY: likely an error: '-0'"
log_always "adjusting NGX_WORKER_PRIORITY=0"
NGX_WORKER_PRIORITY=0
;;
* )
log_always "NGX_WORKER_PRIORITY: unrecognized value: ${NGX_WORKER_PRIORITY}"
log_always "setting NGX_WORKER_PRIORITY=${_NGX_WORKER_PRIORITY}"
@ -38,7 +50,7 @@ esac
[ -n "${NGX_WORKER_RLIMIT_NOFILE:-}" ] || NGX_WORKER_RLIMIT_NOFILE=${_NGX_WORKER_RLIMIT_NOFILE}
case "${NGX_WORKER_RLIMIT_NOFILE}" in
"${_NGX_WORKER_RLIMIT_NOFILE}" ) ;;
[1-9] | [1-9][0-9] )
[0-9] | [1-9][0-9] )
log_always "NGX_WORKER_RLIMIT_NOFILE: too low: ${NGX_WORKER_RLIMIT_NOFILE}"
log_always "setting NGX_WORKER_RLIMIT_NOFILE=${_NGX_WORKER_RLIMIT_NOFILE}"
NGX_WORKER_RLIMIT_NOFILE=${_NGX_WORKER_RLIMIT_NOFILE}
@ -59,7 +71,7 @@ esac
[ -n "${NGX_WORKER_CONNECTIONS:-}" ] || NGX_WORKER_CONNECTIONS=${_NGX_WORKER_CONNECTIONS}
case "${NGX_WORKER_CONNECTIONS}" in
"${_NGX_WORKER_CONNECTIONS}" ) ;;
[1-9] | [1-9][0-9] )
[0-9] | [1-9][0-9] )
log_always "NGX_WORKER_CONNECTIONS: too low: ${NGX_WORKER_CONNECTIONS}"
log_always "setting NGX_WORKER_CONNECTIONS=${_NGX_WORKER_CONNECTIONS}"
NGX_WORKER_CONNECTIONS=${_NGX_WORKER_CONNECTIONS}
@ -82,7 +94,7 @@ nofile_hard=$(ulimit -Hn)
if [ "${nofile_hard}" = unlimited ] ; then
## minor hack (if applicable) :)
nofile_hard=${NGX_WORKER_RLIMIT_NOFILE}
nofile_hard=$((NGX_WORKER_RLIMIT_NOFILE * 2))
fi
nofile_ok=0
@ -104,7 +116,7 @@ if [ ${nofile_ok} = 0 ] ; then
nofile_hard=$(ulimit -Hn)
fi
if [ ${nofile_hard} -lt ${NGX_WORKER_RLIMIT_NOFILE} ] ; then
log_always "lowering NGX_WORKER_RLIMIT_NOFILE to ${nofile_hard}"
log_always "lowering NGX_WORKER_RLIMIT_NOFILE to ${nofile_hard} due to hard limit"
NGX_WORKER_RLIMIT_NOFILE=${nofile_hard}
fi
@ -120,3 +132,15 @@ unset nofile_soft nofile_hard nofile_ok
export NGX_WORKER_PROCESSES NGX_WORKER_PRIORITY NGX_WORKER_RLIMIT_NOFILE NGX_WORKER_CONNECTIONS
unset _NGX_WORKER_PROCESSES _NGX_WORKER_PRIORITY _NGX_WORKER_RLIMIT_NOFILE _NGX_WORKER_CONNECTIONS
if [ ${NGX_WORKER_RLIMIT_NOFILE} -lt ${NGX_WORKER_CONNECTIONS} ] ; then
log_always "WARNING: NGX_WORKER_RLIMIT_NOFILE is less than NGX_WORKER_CONNECTIONS (${NGX_WORKER_RLIMIT_NOFILE} < ${NGX_WORKER_CONNECTIONS})"
else
ratio=$(mawk -v "a=${NGX_WORKER_RLIMIT_NOFILE}" -v "b=${NGX_WORKER_CONNECTIONS}" 'BEGIN{print a/b;exit;}' </dev/null)
case "${ratio}" in
1 | 1.* )
log_always "WARNING: \"NGX_WORKER_RLIMIT_NOFILE/NGX_WORKER_CONNECTIONS\" ratio is too low (=${ratio})"
;;
esac
unset ratio
fi

3
image-entry.d/22-http-max-ranges.envsh
View File

@ -10,9 +10,10 @@ else
[ -n "${NGX_HTTP_MAX_RANGES:-}" ] || NGX_HTTP_MAX_RANGES=${_NGX_HTTP_MAX_RANGES}
case "${NGX_HTTP_MAX_RANGES}" in
"${_NGX_HTTP_MAX_RANGES}" ) ;;
## allow values within [1;999]
[1-9] | [1-9][0-9] | [1-9][0-9][0-9] ) ;;
0 )
log "HTTP: Range/If-Range/Accept-Ranges support is disabled"
log_always "HTTP: Range/If-Range/Accept-Ranges support is disabled by NGX_HTTP_MAX_RANGES=0"
;;
* )
log_always "NGX_HTTP_MAX_RANGES: unrecognized value: ${NGX_HTTP_MAX_RANGES}"

5
image-entry.d/53-expand-templates.sh
View File

@ -3,12 +3,15 @@ set -f
. /image-entry.d/00-common.envsh
## Angie: unset core variable
unset ANGIE ANGIE_BPF_MAPS
[ "${NGX_STRICT_LOAD}" = 0 ] || set -e
cd "${merged_root}/"
expand_error_delim() {
IEP_TRACE=0 log_always ' ----------------------------------- '
IEP_DEBUG=0 log_always ' ----------------------------------- '
}
expand_error() {
[ "${expand_error_seen:-}" != 1 ] || return

2
image-entry.d/54-combine-tree.sh
View File

@ -22,7 +22,7 @@ rm -f "$t" ; unset t
[ "${NGX_STRICT_LOAD}" = 0 ] || set -e
load_error_delim() {
IEP_TRACE=0 log_always ' ----------------------------------- '
IEP_DEBUG=0 log_always ' ----------------------------------- '
}
load_error() {
[ "${load_error_seen:-}" != 1 ] || return

2
image-entry.d/55-remove-merged-tree.sh
View File

@ -3,7 +3,7 @@ set -f
. /image-entry.d/00-common.envsh
if [ "${IEP_TRACE}" = 1 ] ; then
if [ "${IEP_RETAIN_MERGED_TREE}" = 1 ] ; then
log_always "NOT removing merged tree: ${merged_root}/"
else
log "removing merged tree: ${merged_root}/"

3
image-entry.d/90-angie-config-test.sh
View File

@ -3,6 +3,9 @@ set -f
. /image-entry.d/00-common.envsh
## Angie: unset core variable
unset ANGIE ANGIE_BPF_MAPS
## merely debug test
log_always 'test Angie configuration:'
log_always '========================='

49
image-entry.d/99-cleanup-env.envsh
View File

@ -1,41 +1,64 @@
#!/bin/sh
## Angie: unset core variable
unset ANGIE ANGIE_BPF_MAPS
if [ "${IEP_RETAIN_ENV}" = 1 ] ; then
log_always "NOT removing following variables:"
sed -E '/^./s,^, ,' >&2
echo >&2
else
__set="$-"
set +e
if [ "${IEP_TRACE}" = 1 ] ; then
log_always "NOT going to unset following variables:"
sed -E '/^./s,^,- ,' >&2
else
unset __env
unset __env __env_print
while read -r __env ; do
[ -n "${__env}" ] || continue
case "${__env}" in
*\'* )
\'* | \"* )
log "skipping variable (malformed): ${__env}" >&2
continue
;;
esac
if [ "${IEP_DEBUG}" = 1 ] ; then
__env_print="${__env}="$(printenv "${__env}")
__env_print=$(env printf '%q' "${__env_print}")
log_always "unsetting variable: ${__env_print}"
else
log "unsetting variable: ${__env}"
fi
unset "${__env}"
done
unset __env
unset __env __env_print
[ -z "${__set}" ] || set -"${__set}"
unset __set
fi <<-EOF
$(
set +e
cat /proc/self/environ \
| sed -zEn '/^([^=]+).*$/s//\1/p' \
| xargs -0r printf '%q\n' \
| {
f="${target_root}/j2cfg/core-preserve-environment.txt"
## retain variables defined in ".core_worker_env" configuration key
## (if it was specified somewhere in dictionaries - either yaml or json)
f="${target_root}/j2cfg/core-worker-env.txt"
[ -s "$f" ] || exec cat
grep -Fxv -f "$f"
} \
| grep -E \
-e '^(NGX|PYTHON)' \
| {
## remove environment variables:
## 1. variables starting with "NGX" as they are used by configuration templates
## 2. variables containing "_SERVICE" or "_PORT" as they are came from
## container orchestration
grep -E \
-e '^NGX' \
-e '_(SERVICE|PORT)' \
} \
| sort -uV
)
EOF
[ -z "${__set}" ] || set -"${__set}"
unset __set

119
image-entry.sh
View File

@ -1,35 +1,77 @@
#!/bin/sh
set -f
[ -n "${IEP_TRACE}" ] || IEP_TRACE=0
[ "${IEP_TRACE}" = 1 ] || IEP_TRACE=0
[ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): start" >&2
iep_preserve_env() {
## preserve LD_PRELOAD
unset __IEP_LD_PRELOAD
__IEP_LD_PRELOAD="${LD_PRELOAD:-}"
unset LD_PRELOAD
## glibc: preserve GLIBC_TUNABLES
unset __IEP_GLIBC_TUNABLES
__IEP_GLIBC_TUNABLES="${GLIBC_TUNABLES:-}"
unset GLIBC_TUNABLES
## glibc: preserve MALLOC_ARENA_MAX
unset __IEP_MALLOC_ARENA_MAX
__IEP_MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-2}
__IEP_MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-2}"
export MALLOC_ARENA_MAX=2
## jemalloc: preserve MALLOC_CONF
unset __IEP_MALLOC_CONF
__IEP_MALLOC_CONF="${MALLOC_CONF:-}"
unset MALLOC_CONF
}
iep_prepare_env() {
## Angie: unset core variable
unset ANGIE ANGIE_BPF_MAPS
## dumb-init: preserve args
unset IEP_DUMB_INIT_ARGS
IEP_DUMB_INIT_ARGS="${DUMB_INIT_ARGS:-}"
unset DUMB_INIT_ARGS
if [ "${DUMB_INIT_SETSID:-}" = 0 ] ; then
IEP_DUMB_INIT_ARGS="-c${IEP_DUMB_INIT_ARGS:+ }${IEP_DUMB_INIT_ARGS}"
fi
unset DUMB_INIT_SETSID
}
iep_restore_env() {
unset IEP_VERBOSE IEP_TRACE IEP_ROOT IEP_LOCAL_OVERRIDE
unset IEP_DEBUG IEP_VERBOSE IEP_TRACE IEP_ROOT
unset IEP_LOCAL_OVERRIDE IEP_RETAIN_MERGED_TREE IEP_RETAIN_ENV
## restore LD_PRELOAD
if [ -n "${__IEP_LD_PRELOAD}" ] ; then
if [ -n "${__IEP_LD_PRELOAD:-}" ] ; then
export LD_PRELOAD="${__IEP_LD_PRELOAD}"
fi
unset __IEP_LD_PRELOAD
## glibc: restore GLIBC_TUNABLES
if [ -n "${__IEP_GLIBC_TUNABLES:-}" ] ; then
export GLIBC_TUNABLES="${__IEP_GLIBC_TUNABLES}"
fi
unset __IEP_GLIBC_TUNABLES
## glibc: restore MALLOC_ARENA_MAX
if [ "${MALLOC_ARENA_MAX}" = 2 ] ; then
if [ -n "${__IEP_MALLOC_ARENA_MAX:-}" ] ; then
export MALLOC_ARENA_MAX="${__IEP_MALLOC_ARENA_MAX}"
fi
unset __IEP_MALLOC_ARENA_MAX
## jemalloc: restore MALLOC_CONF
if [ -n "${__IEP_MALLOC_CONF:-}" ] ; then
export MALLOC_CONF="${__IEP_MALLOC_CONF}"
fi
unset __IEP_MALLOC_CONF
}
iep_preserve_env
iep_prepare_env
## early setup TMPDIR (affects "mktemp")
export TMPDIR=/run/angie/tmp
@ -39,7 +81,6 @@ export TMPDIR=/run/angie/tmp
# case "$1" in
# angie | */angie ) ;;
# * )
# unset IEP_INIT DUMB_INIT_ARGS
# iep_restore_env
# exec "$@"
# ;;
@ -48,31 +89,46 @@ export TMPDIR=/run/angie/tmp
unset __IEP_SRC ; __IEP_SRC="${0##*/}"
. /image-entry.d/00-common.envsh
unexport IEP_INIT DUMB_INIT_ARGS
IEP_INIT=$(gobool_to_int "${IEP_INIT:-0}" 0)
# unexport IEP_INIT
unset x ; x="${IEP_INIT}" ; unset IEP_INIT ; IEP_INIT="$x" ; unset x
IEP_RETAIN_MERGED_TREE=$(gobool_to_int "${IEP_RETAIN_MERGED_TREE:-0}" 0)
IEP_RETAIN_ENV=$(gobool_to_int "${IEP_RETAIN_ENV:-0}" 0)
export IEP_RETAIN_MERGED_TREE IEP_RETAIN_ENV
# IEP_TRACE=$(gobool_to_int "${IEP_TRACE:-0}" 0)
IEP_DEBUG=$(gobool_to_int "${IEP_DEBUG:-0}" 0)
IEP_VERBOSE=$(gobool_to_int "${IEP_VERBOSE:-${IEP_DEBUG}}" "${IEP_DEBUG}")
export IEP_TRACE IEP_DEBUG IEP_VERBOSE
## run parts (if any)
while read -r f ; do
[ -n "$f" ] || continue
[ -f "$f" ] || continue
unset __IEP_SCRIPT
while read -r __IEP_SCRIPT ; do
[ -n "${__IEP_SCRIPT}" ] || continue
[ -f "${__IEP_SCRIPT}" ] || continue
case "$f" in
case "${__IEP_SCRIPT}" in
*.envsh )
if ! [ -x "$f" ] ; then
log "NOT sourcing $f - not executable"
if ! [ -x "${__IEP_SCRIPT}" ] ; then
log "NOT sourcing ${__IEP_SCRIPT} - not executable"
continue
fi
log_always "sourcing $f"
__IEP_SRC="$f"
. "$f"
[ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): source ${__IEP_SCRIPT}" >&2
log "sourcing ${__IEP_SCRIPT}"
__IEP_SRC="${__IEP_SCRIPT}"
. "${__IEP_SCRIPT}"
__IEP_SRC="${0##*/}"
;;
* )
if ! [ -x "$f" ] ; then
log "NOT running $f - not executable"
if ! [ -x "${__IEP_SCRIPT}" ] ; then
log "NOT running ${__IEP_SCRIPT} - not executable"
continue
fi
log_always "running $f"
"$f"
[ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): run ${__IEP_SCRIPT}" >&2
log "running ${__IEP_SCRIPT}"
"${__IEP_SCRIPT}"
;;
esac
done <<EOF
@ -85,11 +141,30 @@ $(
)
EOF
[ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): end" >&2
if [ "${IEP_DEBUG}" = 1 ] ; then
log_always "ready to run application: $*"
else
log_always "ready to run application"
fi
echo >&2
iep_restore_env
IEP_INIT=$(gobool_to_int "${IEP_INIT:-0}" 0)
## variables that are not so easily unsettable
unset __IEP_ENV
for i in '_' 'SHLVL' ; do
__IEP_ENV="${__IEP_ENV:-}${__IEP_ENV:+ }-u $i"
done
if [ "${IEP_INIT}" = 0 ] ; then
exec "$@"
exec \
${__IEP_ENV:+ env ${__IEP_ENV} } \
"$@"
else
exec dumb-init ${DUMB_INIT_ARGS} "$@"
exec \
${__IEP_ENV:+ env ${__IEP_ENV} } \
dumb-init ${IEP_DUMB_INIT_ARGS} \
"$@"
fi

167
j2cfg/j2cfg/functions.py
View File

@ -5,31 +5,7 @@ import re
import jinja2
def uniq_list(a: list) -> list:
return list(dict.fromkeys(a))
def list_remove_non_str(a: list) -> list:
return list(itertools.filterfalse(
lambda x: not isinstance(x, str), a
))
def list_remove_empty_str(a: list) -> list:
return list(itertools.filterfalse(
lambda x: len(x) == 0, a
))
def uniq_str_list(a: list) -> list:
return uniq_list(list_remove_empty_str(a))
def str_split_to_list(s: str, sep=r'\s+') -> list:
return list_remove_empty_str(
re.split(sep, s)
)
from .settings import is_env_banned
def is_sequence(x) -> bool:
@ -40,33 +16,74 @@ def is_mapping(x) -> bool:
return isinstance(x, collections.abc.Mapping)
def any_to_str_list(k) -> list:
if isinstance(k, str):
return [k]
if is_sequence(k):
return [str(e) for e in k]
return [str(k)]
def uniq(a: (list, set)) -> list:
return list(dict.fromkeys(a))
def is_str_list_re_match(a: list, pattern, flags=0) -> bool:
def remove_non_str(a: (list, set)) -> list:
return list(filter(lambda x: isinstance(x, str), a))
def remove_empty_str(a: (list, set)) -> list:
return list(filter(None, a))
def uniq_str_list(a: (list, set)) -> (list, set):
return remove_empty_str(uniq(a))
def str_split_to_list(s: str, sep=r'\s+') -> list:
return remove_empty_str(re.split(sep, s))
def dict_to_env_str_list(x: dict) -> list:
r = []
for k in sorted(x.keys()):
if x[k] is None:
r.append(f'{k}')
else:
r.append(f'{k}={str(x[k])}')
return r
def any_to_str_list(x) -> list:
if isinstance(x, str):
return [x]
if is_sequence(x):
return [str(e) for e in x]
if is_mapping(x):
return dict_to_env_str_list(x)
return [str(x)]
def is_re_match(a: (list, set), pattern, flags=0) -> bool:
return any(re.match(pattern, x, flags) for x in a)
def is_str_list_re_fullmatch(a: list, pattern, flags=0) -> bool:
def is_re_fullmatch(a: (list, set), pattern, flags=0) -> bool:
return any(re.fullmatch(pattern, x, flags) for x in a)
def str_list_re_match(a: list, pattern, flags=0) -> list:
def re_match(a: (list, set), pattern, flags=0) -> list:
return [x for x in a if re.match(pattern, x, flags)]
def str_list_re_fullmatch(a: list, pattern, flags=0) -> list:
def re_fullmatch(a: (list, set), pattern, flags=0) -> list:
return [x for x in a if re.fullmatch(pattern, x, flags)]
def str_list_re_sub(a: list, pattern, repl, count=0, flags=0) -> list:
def re_match_negate(a: (list, set), pattern, flags=0) -> list:
return [x for x in a if not re.match(pattern, x, flags)]
def re_fullmatch_negate(a: (list, set), pattern, flags=0) -> list:
return [x for x in a if not re.fullmatch(pattern, x, flags)]
def re_sub(a: (list, set), pattern, repl, count=0, flags=0) -> list:
return [re.sub(pattern, repl, x, count, flags) for x in a]
@ -84,9 +101,10 @@ def as_cgi_header(s: str) -> str:
return 'HTTP_' + re.sub('[^A-Z0-9]+', '_', s.upper()).strip('_')
def env_any_to_str_list(x) -> list:
def any_to_env_dict(x) -> dict:
if x is None:
return []
return {}
h = {}
def feed(k, v=None):
@ -96,16 +114,13 @@ def env_any_to_str_list(x) -> list:
if m == '=':
k = k2
v = v2
if len(k) == 0:
return
if not re.fullmatch(r'[a-zA-Z_][a-zA-Z0-9_]*', k):
return
if is_env_banned(k):
return
if k in h:
return
if v is None:
h[k] = v
else:
h[k] = str(v)
h[k] = v if v is None else str(v)
if isinstance(x, str):
feed(x)
@ -113,35 +128,57 @@ def env_any_to_str_list(x) -> list:
for e in x:
feed(e)
elif is_mapping(x):
for k, v in x.items():
feed(k, v)
for k in x:
feed(k, x[k])
else:
return []
return {}
r = []
for k in sorted(h.keys()):
if h[k] is None:
r.append(k)
else:
r.append(f'{k}={h[k]}')
return r
return h
def dict_keys(x: dict) -> list:
return sorted([k for k in x.keys()])
def dict_empty_keys(x: dict) -> list:
return sorted([k for k in x.keys() if x[k] is None])
def dict_non_empty_keys(x: dict) -> list:
return sorted([k for k in x.keys() if x[k] is not None])
def list_diff(a: (list, set), b: (list, set)) -> list:
return list(set(a) - set(b))
def list_intersect(a: (list, set), b: (list, set)) -> list:
return list(set(a) & set(b))
J2CFG_FILTERS = [
any_to_env_dict,
any_to_str_list,
as_cgi_header,
env_any_to_str_list,
dict_empty_keys,
dict_keys,
dict_non_empty_keys,
dict_to_env_str_list,
is_mapping,
is_re_fullmatch,
is_re_match,
is_sequence,
is_str_list_re_fullmatch,
is_str_list_re_match,
list_remove_empty_str,
list_remove_non_str,
list_diff,
list_intersect,
re_fullmatch,
re_fullmatch_negate,
re_match,
re_match_negate,
re_sub,
remove_empty_str,
remove_non_str,
sh_like_file_to_list,
str_list_re_fullmatch,
str_list_re_match,
str_list_re_sub,
str_split_to_list,
uniq_list,
uniq,
uniq_str_list,
]

15
j2cfg/j2cfg/settings.py
View File

@ -1,3 +1,6 @@
import re
J2CFG_TEMPLATE_EXT = '.j2'
J2CFG_PATH = [
@ -24,3 +27,15 @@ J2CFG_JINJA_EXTENSIONS = [
'jinja2.ext.do',
'jinja2.ext.loopcontrols',
]
J2CFG_BANNED_ENVS = [
r'ANGIE(=|$)',
r'ANGIE_BPF_MAPS(=|$)'
]
def is_env_banned(k: str) -> bool:
for r in J2CFG_BANNED_ENVS:
if re.match(r, k):
return True
return False

139
j2cfg/test.j2
View File

@ -1,20 +1,40 @@
j2cfg:
{{ j2cfg }}
{% set x = [1,2,3,4] %}
x = {{ x }}
is_sequence:
{{ x | is_sequence }}
{% set x = {1:2,3:4} %}
x = {{ x }}
is_sequence:
{{ x | is_sequence }}
{% set x = [1,2,3,4] %}
x = {{ x }}
is_mapping:
{{ x | is_mapping }}
{% set x = {1:2,3:4} %}
x = {{ x }}
is_mapping:
{{ x | is_mapping }}
{% set x = [2,3,1,2] %}
x = {{ x }}
uniq_list:
{{ x | uniq_list }}
uniq:
{{ x | uniq }}
{% set x = ['2',3,'1','2'] %}
x = {{ x }}
list_remove_non_str:
{{ x | list_remove_non_str }}
remove_non_str:
{{ x | remove_non_str }}
{% set x = ['2','','1','2'] %}
x = {{ x }}
list_remove_empty_str:
{{ x | list_remove_empty_str }}
remove_empty_str:
{{ x | remove_empty_str }}
{% set x = ['2','3','1','2'] %}
x = {{ x }}
@ -31,25 +51,10 @@ str_split_to_list:
str_split_to_list(':'):
{{ x | str_split_to_list(':') }}
{% set x = [1,2,3,4] %}
{% set x = { 'VAR1': 'Etc/UTC', 'VAR2': '', 'VAR3': None, '4VAR4': 'yeah', 'VAR5=not': 'yeah', 'VAR5=real yeah': None, 'VAR6': {'pi': 3.1415926}, 'VAR7': ['pi', 3.1415926] } %}
x = {{ x }}
is_sequence:
{{ x | is_sequence }}
{% set x = {1:2,3:4} %}
x = {{ x }}
is_sequence:
{{ x | is_sequence }}
{% set x = [1,2,3,4] %}
x = {{ x }}
is_mapping:
{{ x | is_mapping }}
{% set x = {1:2,3:4} %}
x = {{ x }}
is_mapping:
{{ x | is_mapping }}
dict_to_env_str_list:
{{ x | dict_to_env_str_list }}
{% set x = '1 2 3 4' %}
"x = {{ x }}"
@ -68,38 +73,52 @@ any_to_str_list:
{% set x = ['a2','b3','c1','d2'] %}
x = {{ x }}
is_str_list_re_match('[ab]'):
{{ x | is_str_list_re_match('[ab]') }}
is_str_list_re_match('[mn]'):
{{ x | is_str_list_re_match('[mn]') }}
is_re_match('[ab]'):
{{ x | is_re_match('[ab]') }}
is_re_match('[mn]'):
{{ x | is_re_match('[mn]') }}
{% set x = ['a2','b3','c1','d2'] %}
x = {{ x }}
is_str_list_re_fullmatch('[ab]'):
{{ x | is_str_list_re_fullmatch('[ab]') }}
is_str_list_re_fullmatch('[ab][12]'):
{{ x | is_str_list_re_fullmatch('[ab][12]') }}
is_re_fullmatch('[ab]'):
{{ x | is_re_fullmatch('[ab]') }}
is_re_fullmatch('[ab][12]'):
{{ x | is_re_fullmatch('[ab][12]') }}
{% set x = ['a2','b3','c1','d2'] %}
x = {{ x }}
str_list_re_match('[ab]'):
{{ x | str_list_re_match('[ab]') }}
str_list_re_match('[mn]'):
{{ x | str_list_re_match('[mn]') }}
re_match('[ab]'):
{{ x | re_match('[ab]') }}
re_match('[mn]'):
{{ x | re_match('[mn]') }}
{% set x = ['a2','b3','c1','d2'] %}
x = {{ x }}
str_list_re_fullmatch('[ab]'):
{{ x | str_list_re_fullmatch('[ab]') }}
str_list_re_fullmatch('[ab][12]'):
{{ x | str_list_re_fullmatch('[ab][12]') }}
re_fullmatch('[ab]'):
{{ x | re_fullmatch('[ab]') }}
re_fullmatch('[ab][12]'):
{{ x | re_fullmatch('[ab][12]') }}
{% set x = ['a2','b3','c1','d2'] %}
x = {{ x }}
re_match_negate('[ab]'):
{{ x | re_match_negate('[ab]') }}
re_match_negate('[mn]'):
{{ x | re_match_negate('[mn]') }}
{% set x = ['a2','b3','c1','d2'] %}
x = {{ x }}
re_fullmatch_negate('[ab]'):
{{ x | re_fullmatch_negate('[ab]') }}
re_fullmatch_negate('[ab][12]'):
{{ x | re_fullmatch_negate('[ab][12]') }}
{% set x = ['a2b','b3b','c1f','d2g'] %}
x = {{ x }}
str_list_re_sub('[ab]', '_'):
{{ x | str_list_re_sub('[ab]', '_') }}
str_list_re_sub('[mn]', '_'):
{{ x | str_list_re_sub('[mn]', '_') }}
re_sub('[ab]', '_'):
{{ x | re_sub('[ab]', '_') }}
re_sub('[mn]', '_'):
{{ x | re_sub('[mn]', '_') }}
{% set x = 'j2cfg-multi.py' %}
x = {{ x }}
@ -118,15 +137,35 @@ as_cgi_header:
{% set x = 'VAR1=Etc/UTC' %}
x = {{ x }}
env_any_to_str_list:
{{ x | env_any_to_str_list }}
any_to_env_dict:
{{ x | any_to_env_dict }}
{% set x = ['VAR1=Etc/UTC', 'VAR2=', 'VAR3', '4VAR4=yeah', 'VAR5=yeah', 'VAR5=not-yeah'] %}
x = {{ x }}
env_any_to_str_list:
{{ x | env_any_to_str_list }}
any_to_env_dict:
{{ x | any_to_env_dict }}
{% set x = { 'VAR1': 'Etc/UTC', 'VAR2': '', 'VAR3': None, '4VAR4': 'yeah', 'VAR5=not': 'yeah', 'VAR5=real yeah': None, 'VAR6': {'pi': 3.1415926}, 'VAR7': ['pi', 3.1415926] } %}
x = {{ x }}
env_any_to_str_list:
{{ x | env_any_to_str_list }}
any_to_env_dict:
{{ x | any_to_env_dict }}
{% set x = { 'VAR1': 'Etc/UTC', 'VAR2': '', 'VAR3': None, '4VAR4': 'yeah', 'VAR5=not': 'yeah', 'VAR5=real yeah': None, 'VAR6': {'pi': 3.1415926}, 'VAR7': ['pi', 3.1415926] } %}
x = {{ x }}
dict_keys:
{{ x | dict_keys }}
dict_empty_keys:
{{ x | dict_empty_keys }}
dict_non_empty_keys:
{{ x | dict_non_empty_keys }}
{% set x = [1,2,3,4] %}
{% set y = [3,4,5,6] %}
list_diff(x, y):
{{ x | list_diff(y) }}
list_diff(y, x):
{{ y | list_diff(x) }}
list_intersect(x, y):
{{ x | list_intersect(y) }}
list_intersect(y, x):
{{ y | list_intersect(x) }}

2
scripts/envsubst-args.sh
View File

@ -4,7 +4,7 @@ set -f
sed -znE '/^([^=]+)=.*$/s,,\1,p' /proc/self/environ \
| sed -zE \
-e '/^_$/d;/^ENVSUBST_/d;' \
-e '/^__IEP/d;/^IEP_(INIT|VERBOSE|TRACE)$/d' \
-e '/^__IEP_/d;/^IEP_$/d' \
| {
if [ -n "${ENVSUBST_EXCLUDE_REGEX:-}" ] ; then
grep -zEv -e "${ENVSUBST_EXCLUDE_REGEX}"

4
scripts/j2cfg-multi
View File

@ -1,7 +1,7 @@
#!/bin/sh
[ "${IEP_VERBOSE}" = 0 ] || {
[ "${IEP_VERBOSE:-}" = 0 ] || {
pfx=
[ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): "
[ "${IEP_DEBUG:-}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): "
echo "# ${pfx}${0##*/}:" >&2
printf ' - %s\n' "$@" >&2
}

4
scripts/j2cfg-single
View File

@ -1,7 +1,7 @@
#!/bin/sh
[ "${IEP_VERBOSE}" = 0 ] || {
[ "${IEP_VERBOSE:-}" = 0 ] || {
pfx=
[ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): "
[ "${IEP_DEBUG:-}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): "
echo "# ${pfx}${0##*/}:${*:+ $*}" >&2
}
exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@"