rework template unrolling

.dockerignore

@@ -0,0 +1 @@
+jinja2/__pycache__

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+*.py[co]

Dockerfile

@@ -26,9 +26,14 @@ SHELL [ "/bin/sh", "-ec" ]
 COPY /scripts/*       /usr/local/sbin/
 COPY /extra-scripts/* /usr/local/sbin/
 
+COPY /jinja2/ /usr/local/lib/jinja2/
+
 ENV PYTHONDONTWRITEBYTECODE=''
 
 ## Python cache preseed
+
+RUN python3 -m compileall -q -j 2 /usr/local/lib/jinja2/
+
 RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
     find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
     | sed -zEn \
@@ -45,17 +50,14 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
       python3 -m compileall -q -j 2
 
 ## Python cache warmup
-RUN python3 -m site > /dev/null ; \
+RUN echo > /tmp/f.j2 ; \
-    echo > /tmp/f.j2 ; \
+    j2-single /tmp/f.j2 ; \
-    jinja.py /tmp/f.j2 ; \
+    rm -f /tmp/f /tmp/f.j2
-    pip-env.sh pip list -v >/dev/null ; \
-    find "${PYTHON_SITE_PACKAGES}/pip/" -name __pycache__ -exec rm -rf {} +
 
 ## Python cache adjustments
 RUN d="@$(date '+%s')" ; \
-    libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
+    find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
-    find "${libpython}/" -name '*.pyc' -exec touch -m -d "$d" {} + ; \
+    find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
-    find "${libpython}/" -name __pycache__ -exec touch -m -d "$d" {} +
 
 ## ---
 
@@ -71,7 +73,10 @@ COPY --from=certs /usr/local/share/ca-certificates/   /usr/local/share/ca-certif
 
 ## RFC: Python cache
 ## TODO: reduce load by selecting only __pycache__ directories in either way
-# COPY --from=pycache /usr/local/lib/  /usr/local/lib/
+COPY --from=pycache /usr/local/lib/  /usr/local/lib/
+
+## already copied by statement above
+# COPY /jinja2/ /usr/local/lib/jinja2/
 
 ENV ANGIE_MODULES_DIR=/usr/lib/angie/modules
 

image-entry.d/00-common.envsh

@@ -117,55 +117,48 @@ install_userdir() {
 	fi
 }
 
-untemplate_file_envsubst() {
+expand_file_envsubst() {
-	[ -n "$1" ] || return
+	__r=0
-	[ -f "$1" ] || { log_always "file not found: $1" ; return 1 ; }
+	for __src ; do
+		[ -n "${__src}" ] || continue
 
-	[ -n "${NGX_ENVSUBST_SUFFIX:-}" ] || { log "NGX_ENVSUBST_SUFFIX is empty" ; return 1 ; }
+		if ! [ -f "${__src}" ] ; then
+			__r=1
+			log_always "file not found: ${__src}"
+			continue
+		fi
 
-	__dest="$2"
+		case "${__src}" in
-	[ -n "${__dest}" ] || __dest=$(untemplate_path "$1" "${NGX_ENVSUBST_SUFFIX}") || return
+		*.in ) ;;
+		* )
+			__r=1
+			log "expand_file_envsubst: file name extension mismatch: ${__src}"
+			continue
+		;;
+		esac
 
-	if [ -e "${__dest}" ] ; then
+		__dest=$(strip_suffix "${__src}" '.in')
-		log "untemplate_file_envsubst: destination file already exists"
+		if [ -e "${__dest}" ] ; then
-		return
+			__r=1
-	fi
+			log "expand_file_envsubst: destination file already exists: ${__dest}"
+			continue
+		fi
 
-	[ -d "${__dest%/*}" ] || install_userdir "${__dest%/*}" || return
+		log "Running envsubst: ${__src} -> ${__dest}"
-
+		envsubst.sh < "${__src}" > "${__dest}" || __r=1
-	log "Running envsubst: $1 -> ${__dest}"
+	done
-	envsubst.sh < "$1" > "${__dest}" || return
+	unset __src __dest
+	return ${__r}
 }
 
-## notes:
+expand_file_jinja() {
-## - (OPTIONAL) place own wrapper script as "/usr/local/sbin/jinja.py"
+	j2-single "$@" || return $?
-##   in order to perform different template processing
-untemplate_file_jinja() {
-	[ -n "$1" ] || return
-	[ -f "$1" ] || { log_always "file not found: $1" ; return 1 ; }
-
-	[ -n "${NGX_JINJA_SUFFIX:-}" ] || { log "NGX_JINJA_SUFFIX is empty" ; return 1 ; }
-
-	__dest="$2"
-	[ -n "${__dest}" ] || __dest=$(untemplate_path "$1" "${NGX_JINJA_SUFFIX}") || return
-
-	if [ -e "${__dest}" ] ; then
-		log "untemplate_file_jinja: destination file already exists"
-		return
-	fi
-
-	[ -d "${__dest%/*}" ] || install_userdir "${__dest%/*}" || return
-
-	log "Running jinja.py: $1 -> ${__dest}"
-	jinja.py "$1" "${__dest}" || return
 }
 
-untemplate_dir_envsubst() {
+expand_dir_envsubst() {
-	[ -n "${NGX_ENVSUBST_SUFFIX:-}" ] || { log "NGX_ENVSUBST_SUFFIX is empty" ; return 1 ; }
-
 	__template_list=$(mktemp) || return
 
-	find "$@" -follow -type f -name "*${NGX_ENVSUBST_SUFFIX}" \
+	find "$@" -follow -type f -name '*.in' \
 	| sort -uV > "${__template_list}"
 
 	__have_args="${ENVSUBST_ARGS:+1}"
@@ -177,9 +170,10 @@ untemplate_dir_envsubst() {
 		export ENVSUBST_ARGS
 	fi
 
+	__ret=0
 	while read -r __orig_file ; do
 		[ -n "${__orig_file}" ] || continue
-		untemplate_file_envsubst "${__orig_file}"
+		expand_file_envsubst "${__orig_file}" || __ret=1
 	done < "${__template_list}"
 	unset __orig_file
 
@@ -189,23 +183,25 @@ untemplate_dir_envsubst() {
 	unset __have_args
 
 	rm -f "${__template_list}" ; unset __template_list
+	
+	return ${__ret}
 }
 
-untemplate_dir_jinja() {
+expand_dir_jinja() {
-	[ -n "${NGX_JINJA_SUFFIX:-}" ] || { log "NGX_JINJA_SUFFIX is empty" ; return 1 ; }
-
 	__template_list=$(mktemp) || return
 
-	find "$@" -follow -type f -name "*${NGX_JINJA_SUFFIX}" \
+	find "$@" -follow -type f -name '*.j2' -printf '%p\0' \
-	| sort -uV > "${__template_list}"
+	| sort -zuV > "${__template_list}"
 
-	while read -r __orig_file ; do
+	__ret=0
-		[ -n "${__orig_file}" ] || continue
+	if [ -s "${__template_list}" ] ; then
-		untemplate_file_jinja "${__orig_file}"
+		xargs -0r -n 1000 -a "${__template_list}" \
-	done < "${__template_list}"
+		j2-multi < /dev/null || __ret=1
-	unset __orig_file
+	fi
 
 	rm -f "${__template_list}" ; unset __template_list
+
+	return ${__ret}
 }
 
 remap_path() {

image-entry.d/01-defaults.envsh

@@ -18,17 +18,6 @@ NGX_CORE_ENV="${NGX_CORE_ENV:-}"
 
 NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0)
 
-NGX_ENVSUBST_SUFFIX="${NGX_ENVSUBST_SUFFIX:-.in}"
-case "${NGX_ENVSUBST_SUFFIX}" in
-.* ) ;;
-* ) NGX_ENVSUBST_SUFFIX=".${NGX_ENVSUBST_SUFFIX}" ;;
-esac
-NGX_JINJA_SUFFIX="${NGX_JINJA_SUFFIX:-.j2}"
-case "${NGX_JINJA_SUFFIX}" in
-.* ) ;;
-* ) NGX_JINJA_SUFFIX=".${NGX_JINJA_SUFFIX}" ;;
-esac
-
 set +a
 
 if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then

image-entry.d/53-expand-templates.sh

@@ -3,7 +3,15 @@ set -ef
 
 . /image-entry.d/00-common.envsh
 
-untemplate_dir_envsubst "${merged_root}"
+dirs='conf mod modules njs site snip'
-untemplate_dir_jinja    "${merged_root}"
+[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
+
+for n in ${dirs} ; do
+	merged_dir="${merged_root}/$n"
+	[ -d "${merged_dir}" ] || continue
+
+	expand_dir_envsubst "${merged_dir}/"
+	expand_dir_jinja    "${merged_dir}/"
+done
 
 exit 0

jinja2/j2-multi.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import os.path
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import j2common
+
+
+def main():
+    if len(sys.argv) < 2:
+        raise ValueError('not enough arguments (min: 1)')
+
+    j2common.init()
+
+    ret = 0
+    for f in sys.argv[1:]:
+        if not j2common.render_file(f, None, False):
+            ret = 1
+
+    sys.exit(ret)
+
+
+if __name__ == "__main__":
+    main()

jinja2/j2-single.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import os.path
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import j2common
+
+
+def main():
+    if len(sys.argv) < 2:
+        raise ValueError('not enough arguments (min: 1)')
+    if len(sys.argv) > 3:
+        raise ValueError('too many arguments (max: 2)')
+
+    if not sys.argv[1]:
+        raise ValueError('specify input file')
+
+    if len(sys.argv) == 3:
+        if not sys.argv[2]:
+            raise ValueError('specify output file')
+
+    j2common.init()
+
+    j2common.render_file(*sys.argv[1:])
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

jinja2/j2common.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+import importlib
+import json
+import os
+import os.path
+import sys
+import yaml
+import jinja2
+
+ME = sys.argv[0]
+
+J2_MODULES_DEFAULT = 'os os.path sys netaddr psutil re wcmatch'
+J2_SUFFIX = '.j2'
+J2_CFG_PATHS = [
+    '/angie/j2cfg',
+    '/etc/angie/j2cfg',
+]
+J2_CFG_EXTS = [
+    'yml',
+    'yaml',
+    'json',
+]
+J2_SEARCH_PATH = [
+    '/etc/angie',
+    '/run/angie',
+    '/',
+]
+J2_EXT_LIST = [
+    'jinja2.ext.do',
+    'jinja2.ext.loopcontrols',
+]
+
+J2_MODULES = sorted(set(
+    os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ')
+))
+
+J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '')
+
+J2_KWARGS = {}
+
+
+def merge_dict_from_file(filename):
+    if (not filename) or (filename == ''):
+        return False
+    if not os.path.exists(filename):
+        return False
+    if not os.path.isfile(filename):
+        print(
+            f'{ME}: not a file, skipping: {filename}',
+            file=sys.stderr)
+        return False
+    if filename.endswith('.yml') or filename.endswith('.yaml'):
+        with open(filename, mode='r', encoding='utf-8') as fx:
+            x = yaml.safe_load(fx)
+            J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
+        return True
+    if filename.endswith('.json'):
+        with open(filename, mode='r', encoding='utf-8') as fx:
+            x = json.load(fx)
+            J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
+        return True
+    print(
+        f'{ME}: non-recognized name extension: {filename}',
+        file=sys.stderr)
+    return False
+
+
+def merge_dict_default():
+    for base in J2_CFG_PATHS:
+        for full in [base + '.' + ext for ext in J2_CFG_EXTS]:
+            if merge_dict_from_file(full):
+                break
+            continue
+
+
+def render_error(msg, fail=True) -> bool:
+    if fail:
+        raise ValueError(msg)
+    print(f'{ME}: {msg}', file=sys.stderr)
+    return False
+
+
+def render_file(file_in, file_out=None, fail=True):
+    if (not file_in) or (file_in == ''):
+        return render_error(
+            'argument "file_in" is empty',
+            fail)
+    if not os.path.exists(file_in):
+        return render_error(
+            f'file is missing: {file_in}',
+            fail)
+    if not os.path.isfile(file_in):
+        return render_error(
+            f'not a file: {file_in}',
+            fail)
+
+    if file_out:
+        if str(file_in) == str(file_out):
+            return render_error(
+                f'unable to process template inplace: {file_in}',
+                fail)
+        f_out = file_out
+    else:
+        if not file_in.endswith(J2_SUFFIX):
+            return render_error(
+                f'input file name extension mismatch: {file_in}',
+                fail)
+        f_out = os.path.splitext(file_in)[0]
+
+    if os.path.exists(f_out):
+        return render_error(
+            f'output file already exists: {f_out}',
+            fail)
+
+    dirs = J2_SEARCH_PATH.copy()
+    for d in [os.path.dirname(file_in), os.getcwd()]:
+        if d not in dirs:
+            dirs.insert(0, d)
+
+    j2_loader = jinja2.ChoiceLoader([
+        jinja2.FileSystemLoader(
+            d,
+            encoding='utf-8',
+            followlinks=True,
+        ) for d in dirs
+    ])
+
+    j2_environ = jinja2.Environment(
+        loader=j2_loader,
+        extensions=J2_EXT_LIST,
+    )
+    j2_template = j2_environ.get_template(file_in)
+    j2_stream = j2_template.stream(**J2_KWARGS)
+    j2_stream.disable_buffering()
+    j2_stream.dump(f_out, encoding='utf-8')
+    return True
+
+
+def init():
+    global J2_KWARGS
+    for m in J2_MODULES:
+        J2_KWARGS[m] = importlib.import_module(m)
+    J2_KWARGS['env'] = os.environ
+    J2_KWARGS['cfg'] = {}
+
+    if J2_CONFIG != '':
+        if os.path.isfile(J2_CONFIG):
+            merge_dict_from_file(J2_CONFIG)
+        else:
+            print(
+                f'{ME}: J2_CONFIG does not exist, skipping: {J2_CONFIG}',
+                file=sys.stderr
+            )
+            merge_dict_default()
+    else:
+        merge_dict_default()

scripts/j2-multi

@@ -0,0 +1,6 @@
+#!/bin/sh
+[ "${IEP_VERBOSE}" = 0 ] || {
+	echo "Running ${0##*/}:" >&2
+	printf ' - %s\n' "$@" >&2
+}
+exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"

scripts/j2-single

@@ -0,0 +1,5 @@
+#!/bin/sh
+[ "${IEP_VERBOSE}" = 0 ] || {
+	echo "Running ${0##*/}: $*" >&2
+}
+exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"

scripts/jinja.py

@@ -1,119 +0,0 @@
-#!/usr/bin/env python3
-import importlib
-import json
-import os
-import os.path
-import sys
-
-import jinja2
-import yaml
-
-
-J2_MODULES_DEFAULT = 'os os.path sys netaddr psutil re wcmatch'
-J2_SUFFIX_DEFAULT = '.j2'
-J2_CFG_PATHS = [
-    '/angie/jinja',
-    '/etc/angie/jinja',
-]
-J2_CFG_EXTS = [
-    'yml',
-    'yaml',
-    'json',
-]
-J2_SEARCH_PATH = [
-    '.',
-    '/etc/angie',
-]
-J2_EXT_LIST = [
-    'jinja2.ext.do',
-    'jinja2.ext.loopcontrols',
-]
-
-
-J2_MODULES = sorted(set(os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ')))
-
-ME = sys.argv[0]
-
-J2_SUFFIX = os.getenv('NGX_JINJA_SUFFIX', J2_SUFFIX_DEFAULT)
-if J2_SUFFIX == '':
-    raise ValueError('NGX_JINJA_SUFFIX is empty')
-if not J2_SUFFIX.startswith('.'):
-    raise ValueError('NGX_JINJA_SUFFIX does not start with dot (".")')
-
-J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '')
-if (J2_CONFIG != '') and (not os.path.exists(J2_CONFIG)):
-    print(f'{ME}: config does not exist, skipping: {J2_CONFIG}', file=sys.stderr)
-
-
-if len(sys.argv) < 2:
-    raise ValueError('not enough arguments (needed: 2)')
-if len(sys.argv) > 3:
-    raise ValueError('too many arguments (needed: 2)')
-
-if not sys.argv[1]:
-    raise ValueError('specify input file')
-input_file = sys.argv[1]
-if not os.path.exists(input_file):
-    raise ValueError('input file does not exist')
-
-if len(sys.argv) == 3:
-    if not sys.argv[2]:
-        raise ValueError('specify output file')
-    output_file = sys.argv[2]
-else:
-    output_file, ext = os.path.splitext(input_file)
-    if ext != J2_SUFFIX:
-        raise ValueError(f'input file name extension mismatch (not a "{J2_SUFFIX}")')
-
-if input_file == output_file:
-    raise ValueError('unable to process template inplace')
-
-kwargs = {}
-for m in J2_MODULES:
-    kwargs[m] = importlib.import_module(m)
-kwargs['env'] = os.environ
-kwargs['cfg'] = {}
-
-
-def merge_dict_from_file(filename):
-    if not filename:
-        return False
-    if not isinstance(filename, str):
-        return False
-    if filename == '':
-        return False
-    if not os.path.exists(filename):
-        return False
-    if not os.path.isfile(filename):
-        print(f'{ME}: not a file, skipping: {filename}', file=sys.stderr)
-        return False
-    if filename.endswith('.yml') or filename.endswith('.yaml'):
-        with open(filename, mode='r', encoding='utf-8') as fx:
-            x = yaml.safe_load(fx)
-            kwargs['cfg'] = kwargs['cfg'] | x
-        return True
-    if filename.endswith('.json'):
-        with open(filename, mode='r', encoding='utf-8') as fx:
-            x = json.load(fx)
-            kwargs['cfg'] = kwargs['cfg'] | x
-        return True
-    print(f'{ME}: non-recognized name extension: {filename}', file=sys.stderr)
-    return False
-
-
-if (J2_CONFIG != '') and (os.path.isfile(J2_CONFIG)):
-    merge_dict_from_file(J2_CONFIG)
-else:
-    for base in J2_CFG_PATHS:
-        for full in [ base + '.' + ext for ext in J2_CFG_EXTS ]:
-            if merge_dict_from_file(full):
-                break
-            continue
-
-
-j2_loader = jinja2.FileSystemLoader(J2_SEARCH_PATH, followlinks=True)
-j2_environ = jinja2.Environment(loader=j2_loader, extensions=J2_EXT_LIST)
-j2_template = j2_environ.get_template(input_file)
-j2_stream = j2_template.stream(**kwargs)
-j2_stream.disable_buffering()
-j2_stream.dump(output_file, encoding='utf-8')