heavily rework template unrolling

.dockerignore

@@ -1 +1,2 @@
-jinja2/__pycache__
+j2cfg/__pycache__
+j2cfg/j2cfg/__pycache__

Dockerfile

@@ -26,13 +26,13 @@ SHELL [ "/bin/sh", "-ec" ]
 COPY /scripts/*       /usr/local/sbin/
 COPY /extra-scripts/* /usr/local/sbin/
 
-COPY /jinja2/ /usr/local/lib/jinja2/
+COPY /j2cfg/ /usr/local/lib/j2cfg/
 
 ENV PYTHONDONTWRITEBYTECODE=''
 
 ## Python cache preseed
 
-RUN python3 -m compileall -q -j 2 /usr/local/lib/jinja2/
+RUN python3 -m compileall -q -j 2 /usr/local/lib/j2cfg/
 
 RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
     find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
@@ -51,7 +51,7 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
 
 ## Python cache warmup
 RUN echo > /tmp/f.j2 ; \
-    j2-single /tmp/f.j2 ; \
+    j2cfg-single /tmp/f.j2 ; \
     rm -f /tmp/f /tmp/f.j2
 
 ## Python cache adjustments
@@ -76,7 +76,7 @@ COPY --from=certs /usr/local/share/ca-certificates/   /usr/local/share/ca-certif
 COPY --from=pycache /usr/local/lib/  /usr/local/lib/
 
 ## already copied by statement above
-# COPY /jinja2/ /usr/local/lib/jinja2/
+# COPY /j2cfg/ /usr/local/lib/j2cfg/
 
 ENV ANGIE_MODULES_DIR=/usr/lib/angie/modules
 
@@ -141,7 +141,7 @@ RUN install -d -o angie -g angie -m 03777 /angie /run/angie ; \
     ln -sv /run/angie/lock       lock.d ; \
     ln -sv ${ANGIE_MODULES_DIR}  modules.dist ; \
     ## hyper-modular paths:
-    data='conf mod modules njs site snip static' ; \
+    data='conf j2cfg mod modules njs site snip static' ; \
     vardata='cache lib log' ; \
     for n in ${data} ; do \
         for d in "$n" "$n.dist" ; do \

angie/conf.dist/core-preserve-env.conf.j2

@@ -1,33 +1,18 @@
 {#- prologue -#}
-{%- set penv = [] -%}
+{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%}
-{%- if cfg.preserve_env -%}
+{%- set have_tz = preserve_env|is_str_list_re_match('TZ(=|$)') -%}
-  {%- set penv = cfg.preserve_env -%}
+{%- set have_malloc_arena = preserve_env|is_str_list_re_match('MALLOC_ARENA_MAX(=|$)') -%}
-  {%- if penv is string -%}
-    {%- set penv = [penv] -%}
-  {%- elif penv is iterable -%}
-    {#- {%- set penv = penv -%} -#}
-  {%- else -%}
-    {%- set penv = [penv|string()] -%}
-  {%- endif -%}
-{%- endif -%}
-{%- set have = namespace() -%}
-{%- set have.tz = false -%}
-{%- set have.malloc_arena = false -%}
-{#- scan -#}
-{%- for v in penv -%}
-  {%- set have.tz = have.tz or re.match('TZ(=|$)', v|string()) -%}
-  {%- set have.malloc_arena = have.malloc_arena or re.match('MALLOC_ARENA_MAX(=|$)', v|string()) -%}
-{%- endfor -%}
 {#- main part -#}
-{%- if not have.tz -%}
+{%- if not have_tz -%}
 env TZ;
 {% endif %}
-{%- if not have.malloc_arena -%}
+{%- if not have_malloc_arena -%}
 env MALLOC_ARENA_MAX;
 {% endif %}
-{%- for v in penv -%}
+{%- for v in preserve_env -%}
-  {%- if re.search("(\"|'|\\s)", v|string()) %}
+  {%- if re.search("(\"|'|\\s)", v) %}
-env {{ (v|string()).__repr__() }};
+{#- TODO: investigate corrent escape behavior for Angie/nginx -#}
+env {{ v.__repr__() }};
   {%- else %}
 env {{ v }};
   {%- endif %}

angie/j2cfg.dist/compress-types.yml

@@ -0,0 +1,15 @@
+compress_types:
+- application/atom+xml
+- application/javascript
+- application/json
+- application/vnd.api+json
+- application/rss+xml
+- application/x-javascript
+- application/xhtml+xml
+- application/xml
+- image/svg+xml
+- image/x-icon
+- text/css
+- text/javascript
+- text/plain
+- text/xml

angie/j2cfg.dist/core-preserve-environment.txt.j2

@@ -0,0 +1,7 @@
+{#- prologue -#}
+{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%}
+{%- set preserve_vars = preserve_env|str_list_re_fullmatch('[^=]+') -%}
+{#- main part -#}
+{% for v in preserve_vars -%}
+{{ v }}
+{% endfor -%}

angie/snip.dist/brotli/buffers.conf

@@ -0,0 +1,5 @@
+brotli_comp_level   5; # default: 6
+brotli_window     64k; # default: 512k
+
+brotli_min_length     1024;
+brotli_buffers     32  16k;

angie/snip.dist/brotli/gzip.conf

@@ -0,0 +1 @@
+include snip.d/gzip/vary.conf;

angie/snip.dist/brotli/types.conf.j2

@@ -0,0 +1,8 @@
+{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%}
+{%- if mime_types %}
+brotli_types
+{%- for t in mime_types %}
+    {{ t }}
+{%- endfor %}
+;
+{%- endif %}

angie/snip.dist/gzip/buffers.conf

@@ -0,0 +1,4 @@
+gzip_comp_level  2; # default: 1
+
+gzip_min_length     1024;
+gzip_buffers     32  16k;

angie/snip.dist/gzip/proxied.conf

@@ -0,0 +1 @@
+gzip_proxied any;

angie/snip.dist/gzip/types.conf.j2

@@ -0,0 +1,8 @@
+{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%}
+{%- if mime_types %}
+gzip_types
+{%- for t in mime_types %}
+    {{ t }}
+{%- endfor %}
+;
+{%- endif %}

angie/snip.dist/gzip/vary.conf

@@ -0,0 +1 @@
+gzip_vary on;

angie/snip.dist/http-brotli.conf

@@ -1,2 +1,2 @@
-include snip.d/http-brotli.modconf;
+include snip.d/brotli/*.conf;
 brotli on;

angie/snip.dist/http-brotli.modconf.j2

@@ -1,14 +0,0 @@
-{%- from 'mime-types.compress.j2inc' import mime_types with context -%}
-
-## default is 6
-brotli_comp_level  5;
-## default is 512k
-brotli_window 64k;
-
-brotli_min_length  1024;
-brotli_buffers     32  16k;
-
-brotli_types
-## sourced from mime-types.compress.txt
-    {{ mime_types | indent(4) }}
-;

angie/snip.dist/http-gzip.conf

@@ -1,2 +1,2 @@
-include snip.d/http-gzip.modconf;
+include snip.d/gzip/*.conf;
 gzip on;

angie/snip.dist/http-gzip.modconf.j2

@@ -1,15 +0,0 @@
-{%- from 'mime-types.compress.j2inc' import mime_types with context -%}
-
-## default is 1
-gzip_comp_level  2;
-
-gzip_min_length  1024;
-gzip_buffers     32  16k;
-
-gzip_vary     on;
-gzip_proxied  any;
-
-gzip_types
-## sourced from mime-types.compress.txt
-    {{ mime_types | indent(4) }}
-;

angie/snip.dist/http-zstd.conf

@@ -1,2 +1,2 @@
-include snip.d/http-zstd.modconf;
+include snip.d/zstd/*.conf;
 zstd on;

angie/snip.dist/http-zstd.modconf.j2

@@ -1,12 +0,0 @@
-{%- from 'mime-types.compress.j2inc' import mime_types with context -%}
-
-## default is 1
-zstd_comp_level  2;
-
-zstd_min_length  1024;
-zstd_buffers     32  16k;
-
-zstd_types
-## sourced from mime-types.compress.txt
-    {{ mime_types | indent(4) }}
-;

angie/snip.dist/mime-types.compress.j2inc

@@ -1,2 +0,0 @@
-{%- set mime_file = pathlib.Path(os.path.join(os.getenv('NGX_MERGED_ROOT'), 'snip/mime-types.compress.txt')) -%}
-{%- set mime_types = mime_file.read_text() -%}

angie/snip.dist/mime-types.compress.txt

@@ -1,14 +0,0 @@
-application/atom+xml
-application/javascript
-application/json
-application/vnd.api+json
-application/rss+xml
-application/x-javascript
-application/xhtml+xml
-application/xml
-image/svg+xml
-image/x-icon
-text/css
-text/javascript
-text/plain
-text/xml

angie/snip.dist/zstd/buffers.conf

@@ -0,0 +1,4 @@
+zstd_comp_level  2; # default: 1
+
+zstd_min_length  1024;
+zstd_buffers     32  16k;

angie/snip.dist/zstd/gzip.conf

@@ -0,0 +1 @@
+include snip.d/gzip/vary.conf;

angie/snip.dist/zstd/types.conf.j2

@@ -0,0 +1,8 @@
+{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%}
+{%- if mime_types %}
+zstd_types
+{%- for t in mime_types %}
+    {{ t }}
+{%- endfor %}
+;
+{%- endif %}

image-entry.d/00-common.envsh

@@ -82,7 +82,7 @@ untemplate_path() {
 	"${volume_root}"/* | /etc/angie/run/* )
 		strip_suffix "$1" "$2"
 	;;
-	/etc/angie/conf.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* )
+	/etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* )
 		strip_suffix "$1" "$2"
 	;;
 	/etc/angie/static.d/* )
@@ -151,8 +151,8 @@ expand_file_envsubst() {
 	return ${__r}
 }
 
-expand_file_jinja() {
+expand_file_j2cfg() {
-	j2-single "$@" || return $?
+	j2cfg-single "$@" || return $?
 }
 
 expand_dir_envsubst() {
@@ -187,7 +187,7 @@ expand_dir_envsubst() {
 	return ${__ret}
 }
 
-expand_dir_jinja() {
+expand_dir_j2cfg() {
 	__template_list=$(mktemp) || return
 
 	find "$@" -follow -type f -name '*.j2' -printf '%p\0' \
@@ -196,7 +196,7 @@ expand_dir_jinja() {
 	__ret=0
 	if [ -s "${__template_list}" ] ; then
 		xargs -0r -n 1000 -a "${__template_list}" \
-		j2-multi < /dev/null || __ret=1
+		j2cfg-multi < /dev/null || __ret=1
 	fi
 
 	rm -f "${__template_list}" ; unset __template_list
@@ -212,6 +212,10 @@ remap_path() {
 	/etc/angie/conf.dist/* ) echo "${2:-/etc/angie/conf.d}${1#/etc/angie/conf.dist}" ;;
 	/etc/angie/conf/* )      echo "${2:-/etc/angie/conf.d}${1#/etc/angie/conf}" ;;
 	/angie/conf/* )          echo "${2:-/etc/angie/conf.d}${1#/angie/conf}" ;;
+	## j2cfg
+	/etc/angie/j2cfg.dist/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/etc/angie/j2cfg.dist}" ;;
+	/etc/angie/j2cfg/* )      echo "${2:-/etc/angie/j2cfg.d}${1#/etc/angie/j2cfg}" ;;
+	/angie/j2cfg/* )          echo "${2:-/etc/angie/j2cfg.d}${1#/angie/j2cfg}" ;;
 	## mod
 	/etc/angie/mod.dist/* ) echo "${2:-/etc/angie/mod.d}${1#/etc/angie/mod.dist}" ;;
 	/etc/angie/mod/* )      echo "${2:-/etc/angie/mod.d}${1#/etc/angie/mod}" ;;

image-entry.d/01-defaults.envsh

@@ -3,19 +3,12 @@
 ## NB: NGX_DEBUG is set via image build script
 
 set -a
+NGX_STRICT_LOAD=$(gobool_to_int "${NGX_STRICT_LOAD:-1}" 1)
+NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0)
 
 NGX_HTTP=$(gobool_to_int "${NGX_HTTP:-1}" 1)
 NGX_MAIL=$(gobool_to_int "${NGX_MAIL:-0}" 0)
 NGX_STREAM=$(gobool_to_int "${NGX_STREAM:-0}" 0)
-
-NGX_STRICT_LOAD=$(gobool_to_int "${NGX_STRICT_LOAD:-1}" 1)
-
-NGX_CORE_MODULES="${NGX_CORE_MODULES:-}"
-NGX_CORE_EVENTS_SNIPPETS="${NGX_CORE_EVENTS_SNIPPETS:-}"
-NGX_CORE_SNIPPETS="${NGX_CORE_SNIPPETS:-}"
-
-NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0)
-
 set +a
 
 if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then
@@ -24,3 +17,20 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then
 	log_always 'Angie is almost completely TURNED OFF'
 	log_always '======================================'
 fi
+
+unset default_dirs_merge default_dirs_link
+default_dirs_merge='conf j2cfg mod modules njs site snip'
+default_dirs_link=''
+
+if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then
+	NGX_DIRS_MERGE="${NGX_DIRS_MERGE:-} static"
+else
+	NGX_DIRS_LINK="${NGX_DIRS_LINK:-} static"
+fi
+
+set -a
+NGX_DIRS_MERGE=$(sort_dedup_list "${default_dirs_merge} ${NGX_DIRS_MERGE:-}")
+NGX_DIRS_LINK=$(sort_dedup_list "${default_dirs_link} ${NGX_DIRS_LINK:-}")
+set +a
+
+unset default_dirs_merge default_dirs_link

image-entry.d/10-core.envsh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -a
+
+NGX_CORE_MODULES="${NGX_CORE_MODULES:-}"
+NGX_CORE_EVENTS_SNIPPETS="${NGX_CORE_EVENTS_SNIPPETS:-}"
+NGX_CORE_SNIPPETS="${NGX_CORE_SNIPPETS:-}"
+
+set +a

image-entry.d/11-core-modules.envsh

@@ -11,7 +11,7 @@ for i in ${NGX_CORE_MODULES:-} ; do
 
 	if is_builtin_module core "$i" ; then
 		log "$i is builtin module, moving to snippets"
-		core_snippets="${core_snippets}${core_snippets:+ }$i"
+		core_snippets="${core_snippets} $i"
 		continue
 	fi
 
@@ -27,12 +27,10 @@ for i in ${NGX_CORE_MODULES:-} ; do
 done
 unset i
 
-## sort and remove duplicates
-core_snippets=$(sort_dedup_list "${core_snippets}")
-
 set -a
 NGX_CORE_MODULES="${core_modules}"
-NGX_CORE_SNIPPETS="${core_snippets}"
+NGX_CORE_SNIPPETS=$(sort_dedup_list "${core_snippets}")
+NGX_CORE_EVENTS_SNIPPETS=$(sort_dedup_list "${NGX_CORE_EVENTS_SNIPPETS}")
 set +a
 
 unset core_modules core_snippets

image-entry.d/21-http-modules.envsh

@@ -21,7 +21,7 @@ if [ "${NGX_HTTP}" = 1 ] ; then
 
 		if is_builtin_module http "$i" ; then
 			log "$i is builtin module, moving to snippets"
-			http_snippets="${http_snippets}${http_snippets:+ }$i"
+			http_snippets="${http_snippets} $i"
 			continue
 		fi
 
@@ -37,11 +37,9 @@ if [ "${NGX_HTTP}" = 1 ] ; then
 	done
 	unset i
 
-	http_snippets=$(sort_dedup_list "${http_snippets}")
-
 	set -a
 	NGX_HTTP_MODULES="${http_modules}"
-	NGX_HTTP_SNIPPETS="${http_snippets}"
+	NGX_HTTP_SNIPPETS=$(sort_dedup_list "${http_snippets}")
 	set +a
 
 	unset http_modules http_snippets

image-entry.d/31-mail-modules.envsh

@@ -12,7 +12,7 @@ if [ "${NGX_MAIL}" = 1 ] ; then
 
 		if is_builtin_module mail "$i" ; then
 			log "$i is builtin module, moving to snippets"
-			mail_snippets="${mail_snippets}${mail_snippets:+ }$i"
+			mail_snippets="${mail_snippets} $i"
 			continue
 		fi
 
@@ -28,11 +28,9 @@ if [ "${NGX_MAIL}" = 1 ] ; then
 	done
 	unset i
 
-	mail_snippets=$(sort_dedup_list "${mail_snippets}")
-
 	set -a
 	NGX_MAIL_MODULES="${mail_modules}"
-	NGX_MAIL_SNIPPETS="${mail_snippets}"
+	NGX_MAIL_SNIPPETS=$(sort_dedup_list "${mail_snippets}")
 	set +a
 
 	unset mail_modules mail_snippets

image-entry.d/41-stream-modules.envsh

@@ -12,7 +12,7 @@ if [ "${NGX_STREAM}" = 1 ] ; then
 
 		if is_builtin_module stream "$i" ; then
 			log "$i is builtin module, moving to snippets"
-			stream_snippets="${stream_snippets}${stream_snippets:+ }$i"
+			stream_snippets="${stream_snippets} $i"
 			continue
 		fi
 
@@ -28,11 +28,9 @@ if [ "${NGX_STREAM}" = 1 ] ; then
 	done
 	unset i
 
-	stream_snippets=$(sort_dedup_list "${stream_snippets}")
-
 	set -a
 	NGX_STREAM_MODULES="${stream_modules}"
-	NGX_STREAM_SNIPPETS="${stream_snippets}"
+	NGX_STREAM_SNIPPETS=$(sort_dedup_list "${stream_snippets}")
 	set +a
 
 	unset stream_modules stream_snippets

image-entry.d/52-merge-tree.sh

@@ -5,10 +5,9 @@ set -ef
 
 [ -d "${merged_root}" ] || install -d "${merged_root}"
 
-dirs='conf mod modules njs site snip'
+for n in ${NGX_DIRS_MERGE} ; do
-[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
+	[ -n "$n" ] || continue
 
-for n in ${dirs} ; do
 	merged_dir="${merged_root}/$n"
 	while read -r old_path ; do
 		[ -n "${old_path}" ] || continue

image-entry.d/53-expand-templates.sh

@@ -5,31 +5,41 @@ set -f
 
 [ "${NGX_STRICT_LOAD}" = 0 ] || set -e
 
-export NGX_MERGED_ROOT="${merged_root}"
+cd "${merged_root}/"
 
+expand_error_delim() {
+	IEP_TRACE=0 log_always ' ----------------------------------- '
+}
 expand_error() {
-	[ "${expand_error_seen:-}" = 1 ] || log_always 'template expansion has failed'
+	[ "${expand_error_seen:-}" != 1 ] || return
 	expand_error_seen=1
+	expand_error_delim
+	log_always 'template expansion has failed'
 	if [ "${NGX_STRICT_LOAD}" = 1 ] ; then
-		t=10
+		t=15
 		log_always "injecting delay for $t seconds"
+		expand_error_delim
 		sleep $t
 		exit 1
 	fi
+	expand_error_delim
 }
 
-dirs='conf mod modules njs site snip'
-[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
-
 merge_dirs=
-for n in ${dirs} ; do
+for n in ${NGX_DIRS_MERGE} ; do
-	merged_dir="${merged_root}/$n"
+	[ -n "$n" ] || continue
-	[ -d "${merged_dir}" ] || continue
+	[ -d "$n" ] || continue
 
-	merge_dirs="${merge_dirs} ${merged_dir}/"
+	merge_dirs="${merge_dirs} $n/"
 done
 
 expand_dir_envsubst ${merge_dirs} || expand_error
-expand_dir_jinja    ${merge_dirs} || expand_error
+
+set -a
+J2CFG_PATH="${merged_root}/j2cfg"
+J2CFG_SEARCH_PATH="${merged_root}"
+set -a
+
+expand_dir_j2cfg ${merge_dirs} || expand_error
 
 exit 0

image-entry.d/54-combine-tree.sh

@@ -30,7 +30,7 @@ load_error() {
 	load_error_delim
 	log_always 'tree combine has failed'
 	if [ "${NGX_STRICT_LOAD}" = 1 ] ; then
-		t=10
+		t=15
 		log_always "injecting delay for $t seconds"
 		load_error_delim
 		sleep $t
@@ -53,13 +53,11 @@ done
 
 ## provide same symlinks as upstream (both Angie and nginx) docker images do
 d="${target_root}/log"
-[ -e "$d/access.log" ] || ln_s /dev/stdout "$d/access.log"
+[ -e "$d/access.log" ] || ln_s /dev/stdout "$d/access.log" || load_error
-[ -e "$d/error.log" ]  || ln_s /dev/stderr "$d/error.log"
+[ -e "$d/error.log" ]  || ln_s /dev/stderr "$d/error.log" || load_error
 
 ## NB: if any error occurs above then configuration is merely empty and/or broken
 
-dirs='conf mod modules njs site snip'
-[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
 while read -r old_path ; do
 	[ -n "${old_path}" ] || continue
 
@@ -73,7 +71,9 @@ while read -r old_path ; do
 done <<-EOF
 $(
 	set +e
-	for n in ${dirs} ; do
+	for n in ${NGX_DIRS_MERGE} ; do
+		[ -n "$n" ] || continue
+
 		[ -d "${merged_root}/$n" ] || continue
 		find "${merged_root}/$n/" ! -type d
 	done \
@@ -81,17 +81,23 @@ $(
 	  -e "^${merged_root}/(mod|snip)/.+\.load\$" \
 	  -e "^${merged_root}/mod/[^/]+\.preseed\$" \
 	| sort -V
-	set -e
 )
 EOF
 
-if [ "${NGX_PROCESS_STATIC}" = 0 ] ; then
+for n in ${NGX_DIRS_LINK} ; do
-	for d in /angie/static /etc/angie/static /etc/angie/static.dist ; do
+	[ -n "$n" ] || continue
+
+	if [ -e "${target_root}/$n" ] ; then continue ; fi
+	for d in "/angie/$n" "/etc/angie/$n" "/etc/angie/$n.dist" ; do
 		[ -d "$d" ] || continue
-		ln_s "$d" "${target_root}/static"
+		ln_s "$d" "${target_root}/$n"
 		break
 	done
-fi
+
+	[ -d "${target_root}/$n" ] || {
+		log "missing required directory: ${target_root}/$n"
+	}
+done
 
 ## Angie modules are loaded in [strict] order!
 combine_modules() {

image-entry.d/99-cleanup-env.envsh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+__set="$-"
+set +e
+if [ "${IEP_TRACE}" = 1 ] ; then
+	log_always "NOT going to unset following variables:"
+	sed -E '/^./s,^,- ,' >&2
+else
+	unset __env
+	while read -r __env ; do
+		[ -n "${__env}" ] || continue
+
+		case "${__env}" in
+		*\'* )
+			log "skipping variable (malformed): ${__env}" >&2
+			continue
+		;;
+		esac
+
+		log "unsetting variable: ${__env}"
+		unset "${__env}"
+	done
+	unset __env
+fi <<-EOF
+$(
+	cat /proc/self/environ \
+	| sed -zEn '/^([^=]+).*$/s//\1/p' \
+	| xargs -0r printf '%q\n' \
+	| {
+		f="${target_root}/j2cfg/core-preserve-environment.txt"
+		[ -s "$f" ] || exec cat
+		grep -Fxv -f "$f"
+	} \
+	| grep -E \
+	  -e '^(NGX|PYTHON)' \
+	| sort -uV
+)
+EOF
+
+[ -z "${__set}" ] || set -"${__set}"
+unset __set

j2cfg/j2cfg-multi.py

@@ -3,19 +3,18 @@
 import os.path
 import sys
 
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-import j2common
-
 
 def main():
     if len(sys.argv) < 2:
         raise ValueError('not enough arguments (min: 1)')
 
-    j2common.init()
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    import j2cfg
 
+    r = j2cfg.J2cfg(strict=False)
     ret = 0
     for f in sys.argv[1:]:
-        if not j2common.render_file(f, None, False):
+        if not r.render_file(f, None):
             ret = 1
 
     sys.exit(ret)

j2cfg/j2cfg-single.py

@@ -3,9 +3,6 @@
 import os.path
 import sys
 
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-import j2common
-
 
 def main():
     if len(sys.argv) < 2:
@@ -16,13 +13,14 @@ def main():
     if not sys.argv[1]:
         raise ValueError('specify input file')
 
-    if len(sys.argv) == 3:
+    if (len(sys.argv) == 3) and (not sys.argv[2]):
-        if not sys.argv[2]:
+        raise ValueError('specify output file')
-            raise ValueError('specify output file')
 
-    j2common.init()
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    import j2cfg
 
-    j2common.render_file(*sys.argv[1:])
+    r = j2cfg.J2cfg()
+    r.render_file(*sys.argv[1:])
     sys.exit(0)
 
 

j2cfg/j2cfg/__init__.py

@@ -0,0 +1,225 @@
+import importlib
+import json
+import os
+import os.path
+import sys
+
+import jinja2
+import wcmatch.wcmatch
+import yaml
+
+from .functions import *
+from .settings import *
+
+
+J2CFG_CONFIG_EXT = ['yml', 'yaml', 'json']
+
+
+class J2cfg:
+    def __init__(self, strict=True, config=None, config_path=None,
+                 modules=None, search_path=None, template_suffix=None):
+
+        self.strict = strict
+        if not isinstance(self.strict, bool):
+            self.strict = True
+
+        self.config_file = config or os.getenv('J2CFG_CONFIG')
+        if self.config_file is not None:
+            self.config_file = str(self.config_file)
+
+        self.config_path = config_path
+        if self.config_path is None:
+            self.config_path = os.getenv('J2CFG_PATH')
+            if self.config_path is not None:
+                self.config_path = str_split_to_list(self.config_path, ':')
+        if self.config_path is None:
+            self.config_path = J2CFG_PATH.copy()
+        else:
+            self.config_path = any_to_str_list(self.config_path)
+        self.config_path = uniq_str_list(self.config_path)
+
+        self.search_path = search_path
+        if self.search_path is None:
+            self.search_path = os.getenv('J2CFG_SEARCH_PATH')
+            if self.search_path is not None:
+                self.search_path = str_split_to_list(self.search_path, ':')
+        if self.search_path is None:
+            self.search_path = self.config_path.copy()
+        else:
+            self.search_path = any_to_str_list(self.search_path)
+        self.search_path = uniq_str_list(self.search_path)
+        # RFC: should we use the current working directory early?
+        for d in [os.getcwd()]:
+            if d not in self.search_path:
+                self.search_path.insert(0, d)
+
+        self.modules = modules or os.getenv('J2CFG_MODULES')
+        if self.modules is None:
+            self.modules = J2CFG_PYTHON_MODULES.copy()
+        else:
+            if isinstance(self.modules, str):
+                self.modules = str_split_to_list(self.modules)
+            else:
+                self.modules = any_to_str_list(self.modules)
+        self.modules = uniq_str_list(self.modules)
+
+        self.template_suffix = template_suffix or os.getenv('J2CFG_SUFFIX')
+        if self.template_suffix is None:
+            self.template_suffix = J2CFG_TEMPLATE_EXT
+        else:
+            self.template_suffix = str(self.template_suffix)
+            if self.template_suffix == '':
+                self.template_suffix = J2CFG_TEMPLATE_EXT
+        if not self.template_suffix.startswith('.'):
+            self.template_suffix = '.' + self.template_suffix
+
+        self.kwargs = {
+            'env': os.environ,
+            'j2cfg': {}
+        }
+        for m in self.modules:
+            if m in self.kwargs:
+                print(f'J2cfg: kwargs already has {m} key',
+                      file=sys.stderr)
+                continue
+            self.kwargs[m] = importlib.import_module(m)
+
+        def merge_dict_from_file(filename):
+            if filename is None:
+                return False
+            f = str(filename)
+            if f == '':
+                return False
+            if not os.path.exists(f):
+                return False
+            if not os.path.isfile(f):
+                print(
+                    f'J2cfg: not a file, skipping: {filename}',
+                    file=sys.stderr)
+                return False
+
+            if f.endswith('.yml') or f.endswith('.yaml'):
+                with open(f, mode='r', encoding='utf-8') as fx:
+                    x = yaml.safe_load(fx)
+                    self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x
+                return True
+
+            if f.endswith('.json'):
+                with open(f, mode='r', encoding='utf-8') as fx:
+                    x = json.load(fx)
+                    self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x
+                return True
+
+            print(
+                f'J2cfg: non-recognized name extension: {filename}',
+                file=sys.stderr)
+            return False
+
+        def merge_dict_default():
+            search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT])
+            search_flags = wcmatch.wcmatch.RECURSIVE | wcmatch.wcmatch.SYMLINKS
+
+            for d in self.config_path:
+                if not os.path.isdir(d):
+                    continue
+                for f in wcmatch.wcmatch.WcMatch(d, search_pattern,
+                                                 flags=search_flags).imatch():
+                    merge_dict_from_file(f)
+
+        if self.config_file is None:
+            merge_dict_default()
+        else:
+            if os.path.isfile(self.config_file):
+                merge_dict_from_file(self.config_file)
+            else:
+                print(
+                    'J2cfg: J2cfg config file does not exist, skipping: '
+                    + f'{self.config_file}',
+                    file=sys.stderr
+                )
+
+        self.j2fs_loaders = {
+            d: jinja2.FileSystemLoader(
+                d, encoding='utf-8', followlinks=True,
+            ) for d in self.search_path
+        }
+        self.j2env = jinja2.Environment(
+            extensions=J2CFG_JINJA_EXTENSIONS,
+            loader=jinja2.ChoiceLoader([
+                self.j2fs_loaders[d] for d in self.search_path
+            ]),
+        )
+
+        def init_env(e: jinja2.Environment):
+            for s in J2CFG_FILTERS:
+                n = s.__name__
+                if n in e.filters:
+                    print(f'J2cfg: filters already has {n} key',
+                          file=sys.stderr)
+                    continue
+                e.filters[n] = s
+
+        init_env(self.j2env)
+
+    def ensure_fs_loader_for(self, directory: str):
+        if directory in self.j2fs_loaders:
+            return
+        self.j2fs_loaders[directory] = jinja2.FileSystemLoader(
+            directory, encoding='utf-8', followlinks=True,
+        )
+
+    def render_file(self, file_in, file_out=None) -> bool:
+
+        def render_error(msg) -> bool:
+            if self.strict:
+                raise ValueError(msg)
+            print(f'J2cfg: {msg}', file=sys.stderr)
+            return False
+
+        if file_in is None:
+            return render_error(
+                'argument "file_in" is None')
+        f_in = str(file_in)
+        if f_in == '':
+            return render_error(
+                'argument "file_in" is empty')
+        if not os.path.exists(f_in):
+            return render_error(
+                f'file is missing: {file_in}')
+        if not os.path.isfile(f_in):
+            return render_error(
+                f'not a file: {file_in}')
+
+        f_out = file_out
+        if f_out is None:
+            if not f_in.endswith(self.template_suffix):
+                return render_error(
+                    f'input file name extension mismatch: {file_in}')
+            f_out = os.path.splitext(f_in)[0]
+
+        dirs = self.search_path.copy()
+        for d in [os.getcwd(), os.path.dirname(f_in)]:
+            if d in dirs:
+                continue
+            self.ensure_fs_loader_for(d)
+            dirs.insert(0, d)
+
+        j2_environ = self.j2env.overlay(loader=jinja2.ChoiceLoader([
+            self.j2fs_loaders[d] for d in dirs
+        ]))
+        j2_template = j2_environ.get_template(f_in)
+        rendered = j2_template.render(**self.kwargs)
+
+        if os.path.lexists(f_out):
+            if os.path.islink(f_out) or (not os.path.isfile(f_out)):
+                return render_error(
+                    f'output file is not safely writable: {f_out}')
+        if os.path.exists(f_out):
+            if os.path.samefile(f_in, f_out):
+                return render_error(
+                    f'unable to process template inplace: {file_in}')
+
+        with open(f_out, mode='w', encoding='utf-8') as f:
+            f.write(rendered)
+
+        return True

j2cfg/j2cfg/functions.py

@@ -0,0 +1,147 @@
+import collections.abc
+import itertools
+import pathlib
+import re
+
+import jinja2
+
+
+def uniq_list(a: list) -> list:
+    return list(dict.fromkeys(a))
+
+
+def list_remove_non_str(a: list) -> list:
+    return list(itertools.filterfalse(
+        lambda x: not isinstance(x, str), a
+    ))
+
+
+def list_remove_empty_str(a: list) -> list:
+    return list(itertools.filterfalse(
+        lambda x: len(x) == 0, a
+    ))
+
+
+def uniq_str_list(a: list) -> list:
+    return uniq_list(list_remove_empty_str(a))
+
+
+def str_split_to_list(s: str, sep=r'\s+') -> list:
+    return list_remove_empty_str(
+        re.split(sep, s)
+    )
+
+
+def is_sequence(x) -> bool:
+    return isinstance(x, collections.abc.Sequence)
+
+
+def is_mapping(x) -> bool:
+    return isinstance(x, collections.abc.Mapping)
+
+
+def any_to_str_list(k) -> list:
+    if isinstance(k, str):
+        return [k]
+
+    if is_sequence(k):
+        return [str(e) for e in k]
+
+    return [str(k)]
+
+
+def is_str_list_re_match(a: list, pattern, flags=0) -> bool:
+    return any(re.match(pattern, x, flags) for x in a)
+
+
+def is_str_list_re_fullmatch(a: list, pattern, flags=0) -> bool:
+    return any(re.fullmatch(pattern, x, flags) for x in a)
+
+
+def str_list_re_match(a: list, pattern, flags=0) -> list:
+    return [x for x in a if re.match(pattern, x, flags)]
+
+
+def str_list_re_fullmatch(a: list, pattern, flags=0) -> list:
+    return [x for x in a if re.fullmatch(pattern, x, flags)]
+
+
+def str_list_re_sub(a: list, pattern, repl, count=0, flags=0) -> list:
+    return [re.sub(pattern, repl, x, count, flags) for x in a]
+
+
+@jinja2.pass_environment
+def sh_like_file_to_list(j2env, file_in: str) -> list:
+    tpl = j2env.get_template(file_in)
+    text = pathlib.Path(tpl.filename).read_text(encoding='utf-8')
+    lines = re.split(r'\r\n', text)
+    return list(itertools.filterfalse(
+        lambda x: re.match(r'\s*#', x), lines
+    ))
+
+
+def as_cgi_header(s: str) -> str:
+    return 'HTTP_' + re.sub('[^A-Z0-9]+', '_', s.upper()).strip('_')
+
+
+def env_any_to_str_list(x) -> list:
+    if x is None:
+        return []
+    h = {}
+
+    def feed(k, v=None):
+        k = str(k)
+        if v is None:
+            k2, m, v2 = k.partition('=')
+            if m == '=':
+                k = k2
+                v = v2
+        if len(k) == 0:
+            return
+        if not re.fullmatch(r'[a-zA-Z_][a-zA-Z0-9_]*', k):
+            return
+        if k in h:
+            return
+        if v is None:
+            h[k] = v
+        else:
+            h[k] = str(v)
+
+    if isinstance(x, str):
+        feed(x)
+    elif is_sequence(x):
+        for e in x:
+            feed(e)
+    elif is_mapping(x):
+        for k, v in x.items():
+            feed(k, v)
+    else:
+        return []
+
+    r = []
+    for k in sorted(h.keys()):
+        if h[k] is None:
+            r.append(k)
+        else:
+            r.append(f'{k}={h[k]}')
+    return r
+
+
+J2CFG_FILTERS = [
+    any_to_str_list,
+    as_cgi_header,
+    env_any_to_str_list,
+    is_mapping,
+    is_sequence,
+    is_str_list_re_fullmatch,
+    is_str_list_re_match,
+    list_remove_empty_str,
+    list_remove_non_str,
+    sh_like_file_to_list,
+    str_list_re_fullmatch,
+    str_list_re_match,
+    str_list_re_sub,
+    str_split_to_list,
+    uniq_list,
+    uniq_str_list,
+]

j2cfg/j2cfg/settings.py

@@ -0,0 +1,26 @@
+J2CFG_TEMPLATE_EXT = '.j2'
+
+J2CFG_PATH = [
+    '/etc/angie/j2cfg.dist',
+    '/etc/angie/j2cfg',
+    '/angie/j2cfg',
+]
+
+J2CFG_PYTHON_MODULES = [
+    'itertools',
+    'json',
+    'os',
+    'os.path',
+    'pathlib',
+    're',
+    'sys',
+    # installed through pip
+    'psutil',
+    'netaddr',
+    'wcmatch',
+]
+
+J2CFG_JINJA_EXTENSIONS = [
+    'jinja2.ext.do',
+    'jinja2.ext.loopcontrols',
+]

jinja2/j2common.py

@@ -1,159 +0,0 @@
-#!/usr/bin/env python3
-import importlib
-import json
-import os
-import os.path
-import sys
-import yaml
-import jinja2
-
-ME = sys.argv[0]
-
-J2_MODULES_DEFAULT = 'os os.path sys netaddr pathlib psutil re wcmatch'
-J2_SUFFIX = '.j2'
-J2_CFG_PATHS = [
-    '/angie/j2cfg',
-    '/etc/angie/j2cfg',
-]
-J2_CFG_EXTS = [
-    'yml',
-    'yaml',
-    'json',
-]
-J2_SEARCH_PATH = [
-    '/etc/angie',
-    '/run/angie',
-    '/',
-]
-J2_EXT_LIST = [
-    'jinja2.ext.do',
-    'jinja2.ext.loopcontrols',
-]
-
-J2_MODULES = sorted(set(
-    os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ')
-))
-
-J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '')
-
-J2_KWARGS = {}
-
-
-def merge_dict_from_file(filename):
-    if (not filename) or (str(filename) == ''):
-        return False
-    if not os.path.exists(filename):
-        return False
-    if not os.path.isfile(filename):
-        print(
-            f'{ME}: not a file, skipping: {filename}',
-            file=sys.stderr)
-        return False
-    if filename.endswith('.yml') or filename.endswith('.yaml'):
-        with open(filename, mode='r', encoding='utf-8') as fx:
-            x = yaml.safe_load(fx)
-            J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
-        return True
-    if filename.endswith('.json'):
-        with open(filename, mode='r', encoding='utf-8') as fx:
-            x = json.load(fx)
-            J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
-        return True
-    print(
-        f'{ME}: non-recognized name extension: {filename}',
-        file=sys.stderr)
-    return False
-
-
-def merge_dict_default():
-    for base in J2_CFG_PATHS:
-        for full in [base + '.' + ext for ext in J2_CFG_EXTS]:
-            if merge_dict_from_file(full):
-                break
-            continue
-
-
-def render_error(msg, fail=True) -> bool:
-    if fail:
-        raise ValueError(msg)
-    print(f'{ME}: {msg}', file=sys.stderr)
-    return False
-
-
-def render_file(file_in, file_out=None, fail=True):
-    if (not file_in) or (str(file_in) == ''):
-        return render_error(
-            'argument "file_in" is empty',
-            fail)
-    if not os.path.exists(file_in):
-        return render_error(
-            f'file is missing: {file_in}',
-            fail)
-    if not os.path.isfile(file_in):
-        return render_error(
-            f'not a file: {file_in}',
-            fail)
-
-    f_out = file_out
-    if not f_out:
-        if not file_in.endswith(J2_SUFFIX):
-            return render_error(
-                f'input file name extension mismatch: {file_in}',
-                fail)
-        f_out = os.path.splitext(file_in)[0]
-
-    dirs = J2_SEARCH_PATH.copy()
-    for d in [os.path.dirname(file_in), os.getcwd()]:
-        if d not in dirs:
-            dirs.insert(0, d)
-
-    j2_loader = jinja2.ChoiceLoader([
-        jinja2.FileSystemLoader(
-            d,
-            encoding='utf-8',
-            followlinks=True,
-        ) for d in dirs
-    ])
-
-    j2_environ = jinja2.Environment(
-        loader=j2_loader,
-        extensions=J2_EXT_LIST,
-    )
-    j2_template = j2_environ.get_template(file_in)
-    j2_stream = j2_template.stream(**J2_KWARGS)
-    j2_stream.disable_buffering()
-
-    if os.path.lexists(f_out):
-        if os.path.islink(f_out) or (not os.path.isfile(f_out)):
-            return render_error(
-                f'output file is not safely writable: {f_out}',
-                fail)
-    if os.path.exists(f_out):
-        if os.path.samefile(file_in, f_out):
-            return render_error(
-                f'unable to process template inplace: {file_in}',
-                fail)
-    j2_stream.dump(f_out, encoding='utf-8')
-    return True
-
-
-def init():
-    kwa = {}
-    for m in J2_MODULES:
-        kwa[m] = importlib.import_module(m)
-    kwa['env'] = os.environ
-    kwa['cfg'] = {}
-    global J2_KWARGS
-    J2_KWARGS = kwa
-
-    if J2_CONFIG != '':
-        if os.path.isfile(J2_CONFIG):
-            merge_dict_from_file(J2_CONFIG)
-        else:
-            print(
-                f'{ME}: J2_CONFIG does not exist, skipping: {J2_CONFIG}',
-                file=sys.stderr
-            )
-            merge_dict_default()
-    else:
-        merge_dict_default()

scripts/j2cfg-multi

@@ -5,4 +5,4 @@
 	echo "# ${pfx}${0##*/}:" >&2
 	printf ' - %s\n' "$@" >&2
 }
-exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"
+exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@"

scripts/j2cfg-single

@@ -4,4 +4,4 @@
 	[ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): "
 	echo "# ${pfx}${0##*/}:${*:+ $*}" >&2
 }
-exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"
+exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@"