diff --git a/Dockerfile b/Dockerfile index a599a8f..b0965a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN python3 -m compileall -q -j 2 /usr/local/lib/j2cfg/ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \ | sed -zEn \ - -e '/^(asyncio|collections|concurrent|encodings|html|importlib|json|logging|multiprocessing|re|urllib|xml)$/p' \ + -e '/^(collections|importlib|json|re)$/p' \ | sort -zV \ | env -C "${libpython}" xargs -0r \ python3 -m compileall -q -j 2 ; \ @@ -141,7 +141,7 @@ RUN install -d -o angie -g angie -m 03777 /angie /run/angie ; \ ln -sv /run/angie/lock lock.d ; \ ln -sv ${ANGIE_MODULES_DIR} modules.dist ; \ ## hyper-modular paths: - data='conf j2cfg mod modules njs site snip static' ; \ + data='conf j2cfg mod modules site snip static' ; \ vardata='cache lib log' ; \ for n in ${data} ; do \ for d in "$n" "$n.dist" ; do \ diff --git a/angie/conf.dist/core-preserve-env.conf.j2 b/angie/conf.dist/core-preserve-env.conf.j2 deleted file mode 100644 index 8b74b09..0000000 --- a/angie/conf.dist/core-preserve-env.conf.j2 +++ /dev/null @@ -1,19 +0,0 @@ -{#- prologue -#} -{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%} -{%- set have_tz = preserve_env|is_str_list_re_match('TZ(=|$)') -%} -{%- set have_malloc_arena = preserve_env|is_str_list_re_match('MALLOC_ARENA_MAX(=|$)') -%} -{#- main part -#} -{%- if not have_tz -%} -env TZ; -{% endif %} -{%- if not have_malloc_arena -%} -env MALLOC_ARENA_MAX; -{% endif %} -{%- for v in preserve_env -%} - {%- if re.search("(\"|'|\\s)", v) %} -{#- TODO: investigate corrent escape behavior for Angie/nginx -#} -env {{ v.__repr__() }}; - {%- else %} -env {{ v }}; - {%- endif %} -{%- endfor -%} diff --git a/angie/conf.dist/core-worker-env.conf.j2 b/angie/conf.dist/core-worker-env.conf.j2 new file mode 100644 index 0000000..1f2b261 --- /dev/null +++ b/angie/conf.dist/core-worker-env.conf.j2 @@ -0,0 +1,28 @@ +{#- prologue -#} +{#- NB: "TZ" is always provided by Angie itself -#} +{%- set s_vars = ['MALLOC_ARENA_MAX', 'GLIBC_TUNABLES', 'MALLOC_CONF'] -%} +{%- set c_env = ( j2cfg.core_worker_env or [] ) | any_to_env_dict -%} +{%- set c_vars = c_env | dict_keys -%} +{%- set c_vars_preserve = c_env | dict_empty_keys -%} +{%- set c_vars_override = c_env | dict_non_empty_keys -%} +{%- set vars_preserve = ( c_vars_preserve + ( s_vars | list_diff(c_vars) )) | sort -%} +{#- main part -#} +{%- if vars_preserve %} +## preserve + {%- for k in vars_preserve %} +env {{ k }}; + {%- endfor %} +{% endif %} + +{%- if c_vars_override %} +## WARNING! +## explicit environment variables are NOT implemented +## reason: envs are supported only for http_perl but not for http_js/stream_js +## solution: provide environment variables explicitly +## and then list them in "core_worker_env" key in config + {%- for k in c_vars_override %} + {%- set v = c_env[k] -%} +# env {{ k }}={{ v.__repr__() }}; + {%- endif %} + {%- endfor %} +{% endif %} diff --git a/angie/j2cfg.dist/core-preserve-environment.txt.j2 b/angie/j2cfg.dist/core-preserve-environment.txt.j2 deleted file mode 100644 index 6aa02b6..0000000 --- a/angie/j2cfg.dist/core-preserve-environment.txt.j2 +++ /dev/null @@ -1,7 +0,0 @@ -{#- prologue -#} -{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%} -{%- set preserve_vars = preserve_env|str_list_re_fullmatch('[^=]+') -%} -{#- main part -#} -{% for v in preserve_vars -%} -{{ v }} -{% endfor -%} diff --git a/angie/j2cfg.dist/core-worker-env.txt.j2 b/angie/j2cfg.dist/core-worker-env.txt.j2 new file mode 100644 index 0000000..9381896 --- /dev/null +++ b/angie/j2cfg.dist/core-worker-env.txt.j2 @@ -0,0 +1,10 @@ +{#- prologue -#} +{%- set s_vars = ['MALLOC_ARENA_MAX', 'GLIBC_TUNABLES', 'MALLOC_CONF'] -%} +{%- set c_env = ( j2cfg.core_worker_env or [] ) | any_to_env_dict -%} +{%- set c_vars = c_env | dict_keys -%} +{%- set c_vars_preserve = c_env | dict_empty_keys -%} +{%- set vars_preserve = ( c_vars_preserve + ( s_vars | list_diff(c_vars) )) | sort -%} +{#- main part -#} +{%- for k in vars_preserve -%} +{{ k }} +{% endfor -%} diff --git a/image-entry.d/00-common.envsh b/image-entry.d/00-common.envsh index 0aaa51b..7c2dc72 100644 --- a/image-entry.d/00-common.envsh +++ b/image-entry.d/00-common.envsh @@ -12,7 +12,7 @@ have_envvar() { ## unexporting variable in (POSIX) sh is PITA =/ unexport() { - unset ___k ___v + local ___k ___v for ___k ; do [ -n "${___k}" ] || continue have_envvar "${___k}" || continue @@ -20,9 +20,7 @@ unexport() { ___v=$(eval printf '%s' "\"\${${___k}}\"") eval "unset ${___k}" eval "${___k}=$(env printf '%s' \"\${___v}\")" - unset ___v done - unset ___k } ## likely the same as in https://pkg.go.dev/strconv#ParseBool @@ -38,31 +36,33 @@ gobool_to_int() { [ -n "${__IEP_SRC:-}" ] || __IEP_SRC="$0" -IEP_TRACE=$(gobool_to_int "${IEP_TRACE:-0}" 0) -export IEP_TRACE log_always() { - if [ "${IEP_TRACE}" = 1 ] ; then + if [ "${IEP_DEBUG}" = 1 ] ; then echo "# $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): ${__IEP_SRC}${*:+: $*}" else echo "# ${__IEP_SRC}${*:+: $*}" fi >&2 } -IEP_VERBOSE=$(gobool_to_int "${IEP_VERBOSE:-${IEP_TRACE}}" "${IEP_TRACE}") -export IEP_VERBOSE log() { [ "${IEP_VERBOSE}" = 0 ] || log_always "$@" } - log_file() { sed -E '/^./s,^, ,' < "$1" >&2 ; } -if [ "${IEP_VERBOSE}" = 0 ] ; then - ln_s() { ln -s "$@" || return; } - cp_a() { cp -a "$@" || return; } -else - ln_s() { ln -sv "$@" || return; } - cp_a() { cp -av "$@" || return; } -fi +ln_s() { + if [ "${IEP_VERBOSE}" = 0 ] ; then + ln -s "$@" || return + else + ln -sv "$@" || return + fi +} +cp_a() { + if [ "${IEP_VERBOSE}" = 0 ] ; then + cp -a "$@" || return + else + cp -av "$@" || return + fi +} ln_cp() { if [ -h "$1" ] ; then @@ -82,7 +82,7 @@ untemplate_path() { "${volume_root}"/* | /etc/angie/run/* ) strip_suffix "$1" "$2" ;; - /etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* ) + /etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* ) strip_suffix "$1" "$2" ;; /etc/angie/static.d/* ) @@ -118,12 +118,14 @@ install_userdir() { } expand_file_envsubst() { - __r=0 + local __ret __src __dst + + __ret=0 for __src ; do [ -n "${__src}" ] || continue if ! [ -f "${__src}" ] ; then - __r=1 + __ret=1 log_always "file not found: ${__src}" continue fi @@ -131,24 +133,23 @@ expand_file_envsubst() { case "${__src}" in *.in ) ;; * ) - __r=1 + __ret=1 log "expand_file_envsubst: file name extension mismatch: ${__src}" continue ;; esac - __dest=$(strip_suffix "${__src}" '.in') - if [ -e "${__dest}" ] ; then - __r=1 - log "expand_file_envsubst: destination file already exists: ${__dest}" + __dst=$(strip_suffix "${__src}" '.in') + if [ -e "${__dst}" ] ; then + __ret=1 + log "expand_file_envsubst: destination file already exists: ${__dst}" continue fi - log "Running envsubst: ${__src} -> ${__dest}" - envsubst.sh < "${__src}" > "${__dest}" || __r=1 + log "Running envsubst: ${__src} -> ${__dst}" + envsubst.sh < "${__src}" > "${__dst}" || __ret=1 done - unset __src __dest - return ${__r} + return ${__ret} } expand_file_j2cfg() { @@ -156,6 +157,8 @@ expand_file_j2cfg() { } expand_dir_envsubst() { + local __template_list __have_args __ret __orig_file + __template_list=$(mktemp) || return find "$@" -follow -type f -name '*.in' \ @@ -175,7 +178,6 @@ expand_dir_envsubst() { [ -n "${__orig_file}" ] || continue expand_file_envsubst "${__orig_file}" || __ret=1 done < "${__template_list}" - unset __orig_file if [ -z "${__have_args}" ] ; then rm -f "${ENVSUBST_ARGS}" ; unset ENVSUBST_ARGS @@ -188,6 +190,8 @@ expand_dir_envsubst() { } expand_dir_j2cfg() { + local __template_list __ret + __template_list=$(mktemp) || return find "$@" -follow -type f -name '*.j2' -printf '%p\0' \ @@ -224,10 +228,6 @@ remap_path() { /etc/angie/modules.dist/* ) echo "${2:-/etc/angie/modules.d}${1#/etc/angie/modules.dist}" ;; /etc/angie/modules/* ) echo "${2:-/etc/angie/modules.d}${1#/etc/angie/modules}" ;; /angie/modules/* ) echo "${2:-/etc/angie/modules.d}${1#/angie/modules}" ;; - ## njs - /etc/angie/njs.dist/* ) echo "${2:-/etc/angie/njs.d}${1#/etc/angie/njs.dist}" ;; - /etc/angie/njs/* ) echo "${2:-/etc/angie/njs.d}${1#/etc/angie/njs}" ;; - /angie/njs/* ) echo "${2:-/etc/angie/njs.d}${1#/angie/njs}" ;; ## site /etc/angie/site.dist/* ) echo "${2:-/etc/angie/site.d}${1#/etc/angie/site.dist}" ;; /etc/angie/site/* ) echo "${2:-/etc/angie/site.d}${1#/etc/angie/site}" ;; diff --git a/image-entry.d/01-defaults.envsh b/image-entry.d/01-defaults.envsh index e51239f..a975b42 100755 --- a/image-entry.d/01-defaults.envsh +++ b/image-entry.d/01-defaults.envsh @@ -19,7 +19,7 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then fi unset default_dirs_merge default_dirs_link -default_dirs_merge='conf j2cfg mod modules njs site snip' +default_dirs_merge='conf j2cfg mod modules site snip' default_dirs_link='' if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then diff --git a/image-entry.d/04-detect-local-ip-addresses.envsh b/image-entry.d/04-detect-local-ip-addresses.envsh new file mode 100755 index 0000000..bd4a1d6 --- /dev/null +++ b/image-entry.d/04-detect-local-ip-addresses.envsh @@ -0,0 +1,25 @@ +#!/bin/sh + +## allow these addresses to be provided in case of: +## - local development/testing +## - `hostname -I' random failures or misbehavior +if [ -z "${NGX_IP_ADDRESSES:-}" ] ; then + NGX_IP_ADDRESSES=$(hostname -I) +fi +NGX_IP_ADDRESSES=$(printf '%s' "${NGX_IP_ADDRESSES}" | sed -zE 's/^\s+//;s/\s+$//;s/\s+/ /g') +export NGX_IP_ADDRESSES + +unset NGX_IPV4_ADDRESSES NGX_IPV6_ADDRESSES +for i in ${NGX_IP_ADDRESSES} ; do + case "$i" in + *:* ) + NGX_IPV6_ADDRESSES="${NGX_IPV6_ADDRESSES:-}${NGX_IPV6_ADDRESSES:+ }$i" + ;; + * ) + NGX_IPV4_ADDRESSES="${NGX_IPV4_ADDRESSES:-}${NGX_IPV4_ADDRESSES:+ }$i" + ;; + esac +done +unset i + +export NGX_IPV4_ADDRESSES NGX_IPV6_ADDRESSES diff --git a/image-entry.d/13-core-worker-defaults.envsh b/image-entry.d/13-core-worker-defaults.envsh index 915f2ff..a30ecbe 100755 --- a/image-entry.d/13-core-worker-defaults.envsh +++ b/image-entry.d/13-core-worker-defaults.envsh @@ -10,9 +10,16 @@ _NGX_WORKER_CONNECTIONS=4096 [ -n "${NGX_WORKER_PROCESSES:-}" ] || NGX_WORKER_PROCESSES=${_NGX_WORKER_PROCESSES} case "${NGX_WORKER_PROCESSES}" in "${_NGX_WORKER_PROCESSES}" ) ;; -[1-9] | [1-9][0-9] ) ;; -0 | [Aa][Uu][Tt][Oo] ) - log_always "NGX_WORKER_PROCESSES=${NGX_WORKER_PROCESSES} isn't supported yet" +## allow values within [1;999] +[1-9] | [1-9][0-9] | [1-9][0-9][0-9] ) ;; +[Aa][Uu][Tt][Oo] ) + ## adjust + NGX_WORKER_PROCESSES=auto + log_always "NGX_WORKER_PROCESSES: \"auto\" isn't supported by container yet" + log_always "offloading decision to Angie (this could be a problem!)" +;; +0 ) + log_always "NGX_WORKER_PROCESSES: \"0\" isn't supported by container yet" log_always "setting NGX_WORKER_PROCESSES=${_NGX_WORKER_PROCESSES}" NGX_WORKER_PROCESSES=${_NGX_WORKER_PROCESSES} ;; @@ -28,6 +35,11 @@ case "${NGX_WORKER_PRIORITY}" in "${_NGX_WORKER_PRIORITY}" ) ;; -[1-9] | -1[0-9] | -20 ) ;; [0-9] | 1[0-9] | 20 ) ;; + -0 ) + log_always "NGX_WORKER_PRIORITY: likely an error: '-0'" + log_always "adjusting NGX_WORKER_PRIORITY=0" + NGX_WORKER_PRIORITY=0 +;; * ) log_always "NGX_WORKER_PRIORITY: unrecognized value: ${NGX_WORKER_PRIORITY}" log_always "setting NGX_WORKER_PRIORITY=${_NGX_WORKER_PRIORITY}" @@ -38,7 +50,7 @@ esac [ -n "${NGX_WORKER_RLIMIT_NOFILE:-}" ] || NGX_WORKER_RLIMIT_NOFILE=${_NGX_WORKER_RLIMIT_NOFILE} case "${NGX_WORKER_RLIMIT_NOFILE}" in "${_NGX_WORKER_RLIMIT_NOFILE}" ) ;; -[1-9] | [1-9][0-9] ) +[0-9] | [1-9][0-9] ) log_always "NGX_WORKER_RLIMIT_NOFILE: too low: ${NGX_WORKER_RLIMIT_NOFILE}" log_always "setting NGX_WORKER_RLIMIT_NOFILE=${_NGX_WORKER_RLIMIT_NOFILE}" NGX_WORKER_RLIMIT_NOFILE=${_NGX_WORKER_RLIMIT_NOFILE} @@ -59,7 +71,7 @@ esac [ -n "${NGX_WORKER_CONNECTIONS:-}" ] || NGX_WORKER_CONNECTIONS=${_NGX_WORKER_CONNECTIONS} case "${NGX_WORKER_CONNECTIONS}" in "${_NGX_WORKER_CONNECTIONS}" ) ;; -[1-9] | [1-9][0-9] ) +[0-9] | [1-9][0-9] ) log_always "NGX_WORKER_CONNECTIONS: too low: ${NGX_WORKER_CONNECTIONS}" log_always "setting NGX_WORKER_CONNECTIONS=${_NGX_WORKER_CONNECTIONS}" NGX_WORKER_CONNECTIONS=${_NGX_WORKER_CONNECTIONS} @@ -82,7 +94,7 @@ nofile_hard=$(ulimit -Hn) if [ "${nofile_hard}" = unlimited ] ; then ## minor hack (if applicable) :) - nofile_hard=${NGX_WORKER_RLIMIT_NOFILE} + nofile_hard=$((NGX_WORKER_RLIMIT_NOFILE * 2)) fi nofile_ok=0 @@ -104,7 +116,7 @@ if [ ${nofile_ok} = 0 ] ; then nofile_hard=$(ulimit -Hn) fi if [ ${nofile_hard} -lt ${NGX_WORKER_RLIMIT_NOFILE} ] ; then - log_always "lowering NGX_WORKER_RLIMIT_NOFILE to ${nofile_hard}" + log_always "lowering NGX_WORKER_RLIMIT_NOFILE to ${nofile_hard} due to hard limit" NGX_WORKER_RLIMIT_NOFILE=${nofile_hard} fi @@ -120,3 +132,15 @@ unset nofile_soft nofile_hard nofile_ok export NGX_WORKER_PROCESSES NGX_WORKER_PRIORITY NGX_WORKER_RLIMIT_NOFILE NGX_WORKER_CONNECTIONS unset _NGX_WORKER_PROCESSES _NGX_WORKER_PRIORITY _NGX_WORKER_RLIMIT_NOFILE _NGX_WORKER_CONNECTIONS + +if [ ${NGX_WORKER_RLIMIT_NOFILE} -lt ${NGX_WORKER_CONNECTIONS} ] ; then + log_always "WARNING: NGX_WORKER_RLIMIT_NOFILE is less than NGX_WORKER_CONNECTIONS (${NGX_WORKER_RLIMIT_NOFILE} < ${NGX_WORKER_CONNECTIONS})" +else + ratio=$(mawk -v "a=${NGX_WORKER_RLIMIT_NOFILE}" -v "b=${NGX_WORKER_CONNECTIONS}" 'BEGIN{print a/b;exit;}' &2 +## Angie: unset core variable +unset ANGIE ANGIE_BPF_MAPS + +if [ "${IEP_RETAIN_ENV}" = 1 ] ; then + log_always "NOT removing following variables:" + sed -E '/^./s,^, ,' >&2 + echo >&2 else - unset __env + __set="$-" + set +e + + unset __env __env_print while read -r __env ; do [ -n "${__env}" ] || continue case "${__env}" in - *\'* ) + \'* | \"* ) log "skipping variable (malformed): ${__env}" >&2 continue ;; esac - log "unsetting variable: ${__env}" + if [ "${IEP_DEBUG}" = 1 ] ; then + __env_print="${__env}="$(printenv "${__env}") + __env_print=$(env printf '%q' "${__env_print}") + log_always "unsetting variable: ${__env_print}" + else + log "unsetting variable: ${__env}" + fi + unset "${__env}" done - unset __env + unset __env __env_print + + [ -z "${__set}" ] || set -"${__set}" + unset __set fi <<-EOF $( + set +e cat /proc/self/environ \ | sed -zEn '/^([^=]+).*$/s//\1/p' \ | xargs -0r printf '%q\n' \ | { - f="${target_root}/j2cfg/core-preserve-environment.txt" + ## retain variables defined in ".core_worker_env" configuration key + ## (if it was specified somewhere in dictionaries - either yaml or json) + f="${target_root}/j2cfg/core-worker-env.txt" [ -s "$f" ] || exec cat grep -Fxv -f "$f" } \ - | grep -E \ - -e '^(NGX|PYTHON)' \ + | { + ## remove environment variables: + ## 1. variables starting with "NGX" as they are used by configuration templates + ## 2. variables containing "_SERVICE" or "_PORT" as they are came from + ## container orchestration + grep -E \ + -e '^NGX' \ + -e '_(SERVICE|PORT)' \ + + } \ | sort -uV ) EOF - -[ -z "${__set}" ] || set -"${__set}" -unset __set diff --git a/image-entry.sh b/image-entry.sh index 966b044..2c18c00 100755 --- a/image-entry.sh +++ b/image-entry.sh @@ -1,35 +1,77 @@ #!/bin/sh set -f +[ -n "${IEP_TRACE}" ] || IEP_TRACE=0 +[ "${IEP_TRACE}" = 1 ] || IEP_TRACE=0 +[ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): start" >&2 + iep_preserve_env() { ## preserve LD_PRELOAD unset __IEP_LD_PRELOAD __IEP_LD_PRELOAD="${LD_PRELOAD:-}" unset LD_PRELOAD + ## glibc: preserve GLIBC_TUNABLES + unset __IEP_GLIBC_TUNABLES + __IEP_GLIBC_TUNABLES="${GLIBC_TUNABLES:-}" + unset GLIBC_TUNABLES + ## glibc: preserve MALLOC_ARENA_MAX unset __IEP_MALLOC_ARENA_MAX - __IEP_MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-2} + __IEP_MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-2}" export MALLOC_ARENA_MAX=2 + + ## jemalloc: preserve MALLOC_CONF + unset __IEP_MALLOC_CONF + __IEP_MALLOC_CONF="${MALLOC_CONF:-}" + unset MALLOC_CONF +} + +iep_prepare_env() { + ## Angie: unset core variable + unset ANGIE ANGIE_BPF_MAPS + + ## dumb-init: preserve args + unset IEP_DUMB_INIT_ARGS + IEP_DUMB_INIT_ARGS="${DUMB_INIT_ARGS:-}" + unset DUMB_INIT_ARGS + if [ "${DUMB_INIT_SETSID:-}" = 0 ] ; then + IEP_DUMB_INIT_ARGS="-c${IEP_DUMB_INIT_ARGS:+ }${IEP_DUMB_INIT_ARGS}" + fi + unset DUMB_INIT_SETSID } iep_restore_env() { - unset IEP_VERBOSE IEP_TRACE IEP_ROOT IEP_LOCAL_OVERRIDE + unset IEP_DEBUG IEP_VERBOSE IEP_TRACE IEP_ROOT + unset IEP_LOCAL_OVERRIDE IEP_RETAIN_MERGED_TREE IEP_RETAIN_ENV ## restore LD_PRELOAD - if [ -n "${__IEP_LD_PRELOAD}" ] ; then + if [ -n "${__IEP_LD_PRELOAD:-}" ] ; then export LD_PRELOAD="${__IEP_LD_PRELOAD}" fi unset __IEP_LD_PRELOAD + ## glibc: restore GLIBC_TUNABLES + if [ -n "${__IEP_GLIBC_TUNABLES:-}" ] ; then + export GLIBC_TUNABLES="${__IEP_GLIBC_TUNABLES}" + fi + unset __IEP_GLIBC_TUNABLES + ## glibc: restore MALLOC_ARENA_MAX - if [ "${MALLOC_ARENA_MAX}" = 2 ] ; then + if [ -n "${__IEP_MALLOC_ARENA_MAX:-}" ] ; then export MALLOC_ARENA_MAX="${__IEP_MALLOC_ARENA_MAX}" fi unset __IEP_MALLOC_ARENA_MAX + + ## jemalloc: restore MALLOC_CONF + if [ -n "${__IEP_MALLOC_CONF:-}" ] ; then + export MALLOC_CONF="${__IEP_MALLOC_CONF}" + fi + unset __IEP_MALLOC_CONF } iep_preserve_env +iep_prepare_env ## early setup TMPDIR (affects "mktemp") export TMPDIR=/run/angie/tmp @@ -39,7 +81,6 @@ export TMPDIR=/run/angie/tmp # case "$1" in # angie | */angie ) ;; # * ) -# unset IEP_INIT DUMB_INIT_ARGS # iep_restore_env # exec "$@" # ;; @@ -48,31 +89,46 @@ export TMPDIR=/run/angie/tmp unset __IEP_SRC ; __IEP_SRC="${0##*/}" . /image-entry.d/00-common.envsh -unexport IEP_INIT DUMB_INIT_ARGS +IEP_INIT=$(gobool_to_int "${IEP_INIT:-0}" 0) +# unexport IEP_INIT +unset x ; x="${IEP_INIT}" ; unset IEP_INIT ; IEP_INIT="$x" ; unset x + +IEP_RETAIN_MERGED_TREE=$(gobool_to_int "${IEP_RETAIN_MERGED_TREE:-0}" 0) +IEP_RETAIN_ENV=$(gobool_to_int "${IEP_RETAIN_ENV:-0}" 0) +export IEP_RETAIN_MERGED_TREE IEP_RETAIN_ENV + +# IEP_TRACE=$(gobool_to_int "${IEP_TRACE:-0}" 0) +IEP_DEBUG=$(gobool_to_int "${IEP_DEBUG:-0}" 0) +IEP_VERBOSE=$(gobool_to_int "${IEP_VERBOSE:-${IEP_DEBUG}}" "${IEP_DEBUG}") +export IEP_TRACE IEP_DEBUG IEP_VERBOSE ## run parts (if any) -while read -r f ; do - [ -n "$f" ] || continue - [ -f "$f" ] || continue +unset __IEP_SCRIPT +while read -r __IEP_SCRIPT ; do + [ -n "${__IEP_SCRIPT}" ] || continue + [ -f "${__IEP_SCRIPT}" ] || continue - case "$f" in + + case "${__IEP_SCRIPT}" in *.envsh ) - if ! [ -x "$f" ] ; then - log "NOT sourcing $f - not executable" + if ! [ -x "${__IEP_SCRIPT}" ] ; then + log "NOT sourcing ${__IEP_SCRIPT} - not executable" continue fi - log_always "sourcing $f" - __IEP_SRC="$f" - . "$f" + [ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): source ${__IEP_SCRIPT}" >&2 + log "sourcing ${__IEP_SCRIPT}" + __IEP_SRC="${__IEP_SCRIPT}" + . "${__IEP_SCRIPT}" __IEP_SRC="${0##*/}" ;; * ) - if ! [ -x "$f" ] ; then - log "NOT running $f - not executable" + if ! [ -x "${__IEP_SCRIPT}" ] ; then + log "NOT running ${__IEP_SCRIPT} - not executable" continue fi - log_always "running $f" - "$f" + [ "${IEP_TRACE}" = 0 ] || echo "# trace: $(date +'%Y-%m-%d %H:%M:%S.%03N %z'): run ${__IEP_SCRIPT}" >&2 + log "running ${__IEP_SCRIPT}" + "${__IEP_SCRIPT}" ;; esac done <&2 + +if [ "${IEP_DEBUG}" = 1 ] ; then + log_always "ready to run application: $*" +else + log_always "ready to run application" +fi +echo >&2 + iep_restore_env -IEP_INIT=$(gobool_to_int "${IEP_INIT:-0}" 0) +## variables that are not so easily unsettable +unset __IEP_ENV +for i in '_' 'SHLVL' ; do + __IEP_ENV="${__IEP_ENV:-}${__IEP_ENV:+ }-u $i" +done + if [ "${IEP_INIT}" = 0 ] ; then - exec "$@" + exec \ + ${__IEP_ENV:+ env ${__IEP_ENV} } \ + "$@" else - exec dumb-init ${DUMB_INIT_ARGS} "$@" + exec \ + ${__IEP_ENV:+ env ${__IEP_ENV} } \ + dumb-init ${IEP_DUMB_INIT_ARGS} \ + "$@" fi diff --git a/j2cfg/j2cfg/functions.py b/j2cfg/j2cfg/functions.py index fe38019..eccf77a 100644 --- a/j2cfg/j2cfg/functions.py +++ b/j2cfg/j2cfg/functions.py @@ -5,31 +5,7 @@ import re import jinja2 - -def uniq_list(a: list) -> list: - return list(dict.fromkeys(a)) - - -def list_remove_non_str(a: list) -> list: - return list(itertools.filterfalse( - lambda x: not isinstance(x, str), a - )) - - -def list_remove_empty_str(a: list) -> list: - return list(itertools.filterfalse( - lambda x: len(x) == 0, a - )) - - -def uniq_str_list(a: list) -> list: - return uniq_list(list_remove_empty_str(a)) - - -def str_split_to_list(s: str, sep=r'\s+') -> list: - return list_remove_empty_str( - re.split(sep, s) - ) +from .settings import is_env_banned def is_sequence(x) -> bool: @@ -40,33 +16,74 @@ def is_mapping(x) -> bool: return isinstance(x, collections.abc.Mapping) -def any_to_str_list(k) -> list: - if isinstance(k, str): - return [k] - - if is_sequence(k): - return [str(e) for e in k] - - return [str(k)] +def uniq(a: (list, set)) -> list: + return list(dict.fromkeys(a)) -def is_str_list_re_match(a: list, pattern, flags=0) -> bool: +def remove_non_str(a: (list, set)) -> list: + return list(filter(lambda x: isinstance(x, str), a)) + + +def remove_empty_str(a: (list, set)) -> list: + return list(filter(None, a)) + + +def uniq_str_list(a: (list, set)) -> (list, set): + return remove_empty_str(uniq(a)) + + +def str_split_to_list(s: str, sep=r'\s+') -> list: + return remove_empty_str(re.split(sep, s)) + + +def dict_to_env_str_list(x: dict) -> list: + r = [] + for k in sorted(x.keys()): + if x[k] is None: + r.append(f'{k}') + else: + r.append(f'{k}={str(x[k])}') + return r + + +def any_to_str_list(x) -> list: + if isinstance(x, str): + return [x] + + if is_sequence(x): + return [str(e) for e in x] + + if is_mapping(x): + return dict_to_env_str_list(x) + + return [str(x)] + + +def is_re_match(a: (list, set), pattern, flags=0) -> bool: return any(re.match(pattern, x, flags) for x in a) -def is_str_list_re_fullmatch(a: list, pattern, flags=0) -> bool: +def is_re_fullmatch(a: (list, set), pattern, flags=0) -> bool: return any(re.fullmatch(pattern, x, flags) for x in a) -def str_list_re_match(a: list, pattern, flags=0) -> list: +def re_match(a: (list, set), pattern, flags=0) -> list: return [x for x in a if re.match(pattern, x, flags)] -def str_list_re_fullmatch(a: list, pattern, flags=0) -> list: +def re_fullmatch(a: (list, set), pattern, flags=0) -> list: return [x for x in a if re.fullmatch(pattern, x, flags)] -def str_list_re_sub(a: list, pattern, repl, count=0, flags=0) -> list: +def re_match_negate(a: (list, set), pattern, flags=0) -> list: + return [x for x in a if not re.match(pattern, x, flags)] + + +def re_fullmatch_negate(a: (list, set), pattern, flags=0) -> list: + return [x for x in a if not re.fullmatch(pattern, x, flags)] + + +def re_sub(a: (list, set), pattern, repl, count=0, flags=0) -> list: return [re.sub(pattern, repl, x, count, flags) for x in a] @@ -84,9 +101,10 @@ def as_cgi_header(s: str) -> str: return 'HTTP_' + re.sub('[^A-Z0-9]+', '_', s.upper()).strip('_') -def env_any_to_str_list(x) -> list: +def any_to_env_dict(x) -> dict: if x is None: - return [] + return {} + h = {} def feed(k, v=None): @@ -96,16 +114,13 @@ def env_any_to_str_list(x) -> list: if m == '=': k = k2 v = v2 - if len(k) == 0: - return if not re.fullmatch(r'[a-zA-Z_][a-zA-Z0-9_]*', k): return + if is_env_banned(k): + return if k in h: return - if v is None: - h[k] = v - else: - h[k] = str(v) + h[k] = v if v is None else str(v) if isinstance(x, str): feed(x) @@ -113,35 +128,57 @@ def env_any_to_str_list(x) -> list: for e in x: feed(e) elif is_mapping(x): - for k, v in x.items(): - feed(k, v) + for k in x: + feed(k, x[k]) else: - return [] + return {} - r = [] - for k in sorted(h.keys()): - if h[k] is None: - r.append(k) - else: - r.append(f'{k}={h[k]}') - return r + return h + + +def dict_keys(x: dict) -> list: + return sorted([k for k in x.keys()]) + + +def dict_empty_keys(x: dict) -> list: + return sorted([k for k in x.keys() if x[k] is None]) + + +def dict_non_empty_keys(x: dict) -> list: + return sorted([k for k in x.keys() if x[k] is not None]) + + +def list_diff(a: (list, set), b: (list, set)) -> list: + return list(set(a) - set(b)) + + +def list_intersect(a: (list, set), b: (list, set)) -> list: + return list(set(a) & set(b)) J2CFG_FILTERS = [ + any_to_env_dict, any_to_str_list, as_cgi_header, - env_any_to_str_list, + dict_empty_keys, + dict_keys, + dict_non_empty_keys, + dict_to_env_str_list, is_mapping, + is_re_fullmatch, + is_re_match, is_sequence, - is_str_list_re_fullmatch, - is_str_list_re_match, - list_remove_empty_str, - list_remove_non_str, + list_diff, + list_intersect, + re_fullmatch, + re_fullmatch_negate, + re_match, + re_match_negate, + re_sub, + remove_empty_str, + remove_non_str, sh_like_file_to_list, - str_list_re_fullmatch, - str_list_re_match, - str_list_re_sub, str_split_to_list, - uniq_list, + uniq, uniq_str_list, ] diff --git a/j2cfg/j2cfg/settings.py b/j2cfg/j2cfg/settings.py index 7bb9eeb..4c1579f 100644 --- a/j2cfg/j2cfg/settings.py +++ b/j2cfg/j2cfg/settings.py @@ -1,3 +1,6 @@ +import re + + J2CFG_TEMPLATE_EXT = '.j2' J2CFG_PATH = [ @@ -24,3 +27,15 @@ J2CFG_JINJA_EXTENSIONS = [ 'jinja2.ext.do', 'jinja2.ext.loopcontrols', ] + +J2CFG_BANNED_ENVS = [ + r'ANGIE(=|$)', + r'ANGIE_BPF_MAPS(=|$)' +] + + +def is_env_banned(k: str) -> bool: + for r in J2CFG_BANNED_ENVS: + if re.match(r, k): + return True + return False diff --git a/j2cfg/test.j2 b/j2cfg/test.j2 index e730ada..d774128 100644 --- a/j2cfg/test.j2 +++ b/j2cfg/test.j2 @@ -1,20 +1,40 @@ j2cfg: {{ j2cfg }} +{% set x = [1,2,3,4] %} +x = {{ x }} +is_sequence: +{{ x | is_sequence }} + +{% set x = {1:2,3:4} %} +x = {{ x }} +is_sequence: +{{ x | is_sequence }} + +{% set x = [1,2,3,4] %} +x = {{ x }} +is_mapping: +{{ x | is_mapping }} + +{% set x = {1:2,3:4} %} +x = {{ x }} +is_mapping: +{{ x | is_mapping }} + {% set x = [2,3,1,2] %} x = {{ x }} -uniq_list: -{{ x | uniq_list }} +uniq: +{{ x | uniq }} {% set x = ['2',3,'1','2'] %} x = {{ x }} -list_remove_non_str: -{{ x | list_remove_non_str }} +remove_non_str: +{{ x | remove_non_str }} {% set x = ['2','','1','2'] %} x = {{ x }} -list_remove_empty_str: -{{ x | list_remove_empty_str }} +remove_empty_str: +{{ x | remove_empty_str }} {% set x = ['2','3','1','2'] %} x = {{ x }} @@ -31,25 +51,10 @@ str_split_to_list: str_split_to_list(':'): {{ x | str_split_to_list(':') }} -{% set x = [1,2,3,4] %} +{% set x = { 'VAR1': 'Etc/UTC', 'VAR2': '', 'VAR3': None, '4VAR4': 'yeah', 'VAR5=not': 'yeah', 'VAR5=real yeah': None, 'VAR6': {'pi': 3.1415926}, 'VAR7': ['pi', 3.1415926] } %} x = {{ x }} -is_sequence: -{{ x | is_sequence }} - -{% set x = {1:2,3:4} %} -x = {{ x }} -is_sequence: -{{ x | is_sequence }} - -{% set x = [1,2,3,4] %} -x = {{ x }} -is_mapping: -{{ x | is_mapping }} - -{% set x = {1:2,3:4} %} -x = {{ x }} -is_mapping: -{{ x | is_mapping }} +dict_to_env_str_list: +{{ x | dict_to_env_str_list }} {% set x = '1 2 3 4' %} "x = {{ x }}" @@ -68,38 +73,52 @@ any_to_str_list: {% set x = ['a2','b3','c1','d2'] %} x = {{ x }} -is_str_list_re_match('[ab]'): -{{ x | is_str_list_re_match('[ab]') }} -is_str_list_re_match('[mn]'): -{{ x | is_str_list_re_match('[mn]') }} +is_re_match('[ab]'): +{{ x | is_re_match('[ab]') }} +is_re_match('[mn]'): +{{ x | is_re_match('[mn]') }} {% set x = ['a2','b3','c1','d2'] %} x = {{ x }} -is_str_list_re_fullmatch('[ab]'): -{{ x | is_str_list_re_fullmatch('[ab]') }} -is_str_list_re_fullmatch('[ab][12]'): -{{ x | is_str_list_re_fullmatch('[ab][12]') }} +is_re_fullmatch('[ab]'): +{{ x | is_re_fullmatch('[ab]') }} +is_re_fullmatch('[ab][12]'): +{{ x | is_re_fullmatch('[ab][12]') }} {% set x = ['a2','b3','c1','d2'] %} x = {{ x }} -str_list_re_match('[ab]'): -{{ x | str_list_re_match('[ab]') }} -str_list_re_match('[mn]'): -{{ x | str_list_re_match('[mn]') }} +re_match('[ab]'): +{{ x | re_match('[ab]') }} +re_match('[mn]'): +{{ x | re_match('[mn]') }} {% set x = ['a2','b3','c1','d2'] %} x = {{ x }} -str_list_re_fullmatch('[ab]'): -{{ x | str_list_re_fullmatch('[ab]') }} -str_list_re_fullmatch('[ab][12]'): -{{ x | str_list_re_fullmatch('[ab][12]') }} +re_fullmatch('[ab]'): +{{ x | re_fullmatch('[ab]') }} +re_fullmatch('[ab][12]'): +{{ x | re_fullmatch('[ab][12]') }} + +{% set x = ['a2','b3','c1','d2'] %} +x = {{ x }} +re_match_negate('[ab]'): +{{ x | re_match_negate('[ab]') }} +re_match_negate('[mn]'): +{{ x | re_match_negate('[mn]') }} + +{% set x = ['a2','b3','c1','d2'] %} +x = {{ x }} +re_fullmatch_negate('[ab]'): +{{ x | re_fullmatch_negate('[ab]') }} +re_fullmatch_negate('[ab][12]'): +{{ x | re_fullmatch_negate('[ab][12]') }} {% set x = ['a2b','b3b','c1f','d2g'] %} x = {{ x }} -str_list_re_sub('[ab]', '_'): -{{ x | str_list_re_sub('[ab]', '_') }} -str_list_re_sub('[mn]', '_'): -{{ x | str_list_re_sub('[mn]', '_') }} +re_sub('[ab]', '_'): +{{ x | re_sub('[ab]', '_') }} +re_sub('[mn]', '_'): +{{ x | re_sub('[mn]', '_') }} {% set x = 'j2cfg-multi.py' %} x = {{ x }} @@ -118,15 +137,35 @@ as_cgi_header: {% set x = 'VAR1=Etc/UTC' %} x = {{ x }} -env_any_to_str_list: -{{ x | env_any_to_str_list }} +any_to_env_dict: +{{ x | any_to_env_dict }} {% set x = ['VAR1=Etc/UTC', 'VAR2=', 'VAR3', '4VAR4=yeah', 'VAR5=yeah', 'VAR5=not-yeah'] %} x = {{ x }} -env_any_to_str_list: -{{ x | env_any_to_str_list }} +any_to_env_dict: +{{ x | any_to_env_dict }} {% set x = { 'VAR1': 'Etc/UTC', 'VAR2': '', 'VAR3': None, '4VAR4': 'yeah', 'VAR5=not': 'yeah', 'VAR5=real yeah': None, 'VAR6': {'pi': 3.1415926}, 'VAR7': ['pi', 3.1415926] } %} x = {{ x }} -env_any_to_str_list: -{{ x | env_any_to_str_list }} +any_to_env_dict: +{{ x | any_to_env_dict }} + +{% set x = { 'VAR1': 'Etc/UTC', 'VAR2': '', 'VAR3': None, '4VAR4': 'yeah', 'VAR5=not': 'yeah', 'VAR5=real yeah': None, 'VAR6': {'pi': 3.1415926}, 'VAR7': ['pi', 3.1415926] } %} +x = {{ x }} +dict_keys: +{{ x | dict_keys }} +dict_empty_keys: +{{ x | dict_empty_keys }} +dict_non_empty_keys: +{{ x | dict_non_empty_keys }} + +{% set x = [1,2,3,4] %} +{% set y = [3,4,5,6] %} +list_diff(x, y): +{{ x | list_diff(y) }} +list_diff(y, x): +{{ y | list_diff(x) }} +list_intersect(x, y): +{{ x | list_intersect(y) }} +list_intersect(y, x): +{{ y | list_intersect(x) }} diff --git a/scripts/envsubst-args.sh b/scripts/envsubst-args.sh index d318c87..27dbaef 100755 --- a/scripts/envsubst-args.sh +++ b/scripts/envsubst-args.sh @@ -4,7 +4,7 @@ set -f sed -znE '/^([^=]+)=.*$/s,,\1,p' /proc/self/environ \ | sed -zE \ -e '/^_$/d;/^ENVSUBST_/d;' \ - -e '/^__IEP/d;/^IEP_(INIT|VERBOSE|TRACE)$/d' \ + -e '/^__IEP_/d;/^IEP_$/d' \ | { if [ -n "${ENVSUBST_EXCLUDE_REGEX:-}" ] ; then grep -zEv -e "${ENVSUBST_EXCLUDE_REGEX}" diff --git a/scripts/j2cfg-multi b/scripts/j2cfg-multi index 9102305..849046c 100755 --- a/scripts/j2cfg-multi +++ b/scripts/j2cfg-multi @@ -1,7 +1,7 @@ #!/bin/sh -[ "${IEP_VERBOSE}" = 0 ] || { +[ "${IEP_VERBOSE:-}" = 0 ] || { pfx= - [ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): " + [ "${IEP_DEBUG:-}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): " echo "# ${pfx}${0##*/}:" >&2 printf ' - %s\n' "$@" >&2 } diff --git a/scripts/j2cfg-single b/scripts/j2cfg-single index f2a8d5a..eacddd0 100755 --- a/scripts/j2cfg-single +++ b/scripts/j2cfg-single @@ -1,7 +1,7 @@ #!/bin/sh -[ "${IEP_VERBOSE}" = 0 ] || { +[ "${IEP_VERBOSE:-}" = 0 ] || { pfx= - [ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): " + [ "${IEP_DEBUG:-}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): " echo "# ${pfx}${0##*/}:${*:+ $*}" >&2 } exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@" \ No newline at end of file