diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1df2d86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +jinja2/__pycache__ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..45d55a0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +*.py[co] diff --git a/Dockerfile b/Dockerfile index f09a331..1297270 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,14 @@ SHELL [ "/bin/sh", "-ec" ] COPY /scripts/* /usr/local/sbin/ COPY /extra-scripts/* /usr/local/sbin/ +COPY /jinja2/ /usr/local/lib/jinja2/ + ENV PYTHONDONTWRITEBYTECODE='' ## Python cache preseed + +RUN python3 -m compileall -q -j 2 /usr/local/lib/jinja2/ + RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \ | sed -zEn \ @@ -45,17 +50,14 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ python3 -m compileall -q -j 2 ## Python cache warmup -RUN python3 -m site > /dev/null ; \ - echo > /tmp/f.j2 ; \ - jinja.py /tmp/f.j2 ; \ - pip-env.sh pip list -v >/dev/null ; \ - find "${PYTHON_SITE_PACKAGES}/pip/" -name __pycache__ -exec rm -rf {} + +RUN echo > /tmp/f.j2 ; \ + j2-single /tmp/f.j2 ; \ + rm -f /tmp/f /tmp/f.j2 ## Python cache adjustments RUN d="@$(date '+%s')" ; \ - libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ - find "${libpython}/" -name '*.pyc' -exec touch -m -d "$d" {} + ; \ - find "${libpython}/" -name __pycache__ -exec touch -m -d "$d" {} + + find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \ + find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} + ## --- @@ -71,7 +73,10 @@ COPY --from=certs /usr/local/share/ca-certificates/ /usr/local/share/ca-certif ## RFC: Python cache ## TODO: reduce load by selecting only __pycache__ directories in either way -# COPY --from=pycache /usr/local/lib/ /usr/local/lib/ +COPY --from=pycache /usr/local/lib/ /usr/local/lib/ + +## already copied by statement above +# COPY /jinja2/ /usr/local/lib/jinja2/ ENV ANGIE_MODULES_DIR=/usr/lib/angie/modules diff --git a/image-entry.d/00-common.envsh b/image-entry.d/00-common.envsh index 64ba8e9..1bbb682 100644 --- a/image-entry.d/00-common.envsh +++ b/image-entry.d/00-common.envsh @@ -117,55 +117,48 @@ install_userdir() { fi } -untemplate_file_envsubst() { - [ -n "$1" ] || return - [ -f "$1" ] || { log_always "file not found: $1" ; return 1 ; } +expand_file_envsubst() { + __r=0 + for __src ; do + [ -n "${__src}" ] || continue - [ -n "${NGX_ENVSUBST_SUFFIX:-}" ] || { log "NGX_ENVSUBST_SUFFIX is empty" ; return 1 ; } + if ! [ -f "${__src}" ] ; then + __r=1 + log_always "file not found: ${__src}" + continue + fi - __dest="$2" - [ -n "${__dest}" ] || __dest=$(untemplate_path "$1" "${NGX_ENVSUBST_SUFFIX}") || return + case "${__src}" in + *.in ) ;; + * ) + __r=1 + log "expand_file_envsubst: file name extension mismatch: ${__src}" + continue + ;; + esac - if [ -e "${__dest}" ] ; then - log "untemplate_file_envsubst: destination file already exists" - return - fi + __dest=$(strip_suffix "${__src}" '.in') + if [ -e "${__dest}" ] ; then + __r=1 + log "expand_file_envsubst: destination file already exists: ${__dest}" + continue + fi - [ -d "${__dest%/*}" ] || install_userdir "${__dest%/*}" || return - - log "Running envsubst: $1 -> ${__dest}" - envsubst.sh < "$1" > "${__dest}" || return + log "Running envsubst: ${__src} -> ${__dest}" + envsubst.sh < "${__src}" > "${__dest}" || __r=1 + done + unset __src __dest + return ${__r} } -## notes: -## - (OPTIONAL) place own wrapper script as "/usr/local/sbin/jinja.py" -## in order to perform different template processing -untemplate_file_jinja() { - [ -n "$1" ] || return - [ -f "$1" ] || { log_always "file not found: $1" ; return 1 ; } - - [ -n "${NGX_JINJA_SUFFIX:-}" ] || { log "NGX_JINJA_SUFFIX is empty" ; return 1 ; } - - __dest="$2" - [ -n "${__dest}" ] || __dest=$(untemplate_path "$1" "${NGX_JINJA_SUFFIX}") || return - - if [ -e "${__dest}" ] ; then - log "untemplate_file_jinja: destination file already exists" - return - fi - - [ -d "${__dest%/*}" ] || install_userdir "${__dest%/*}" || return - - log "Running jinja.py: $1 -> ${__dest}" - jinja.py "$1" "${__dest}" || return +expand_file_jinja() { + j2-single "$@" || return $? } -untemplate_dir_envsubst() { - [ -n "${NGX_ENVSUBST_SUFFIX:-}" ] || { log "NGX_ENVSUBST_SUFFIX is empty" ; return 1 ; } - +expand_dir_envsubst() { __template_list=$(mktemp) || return - find "$@" -follow -type f -name "*${NGX_ENVSUBST_SUFFIX}" \ + find "$@" -follow -type f -name '*.in' \ | sort -uV > "${__template_list}" __have_args="${ENVSUBST_ARGS:+1}" @@ -177,9 +170,10 @@ untemplate_dir_envsubst() { export ENVSUBST_ARGS fi + __ret=0 while read -r __orig_file ; do [ -n "${__orig_file}" ] || continue - untemplate_file_envsubst "${__orig_file}" + expand_file_envsubst "${__orig_file}" || __ret=1 done < "${__template_list}" unset __orig_file @@ -189,23 +183,25 @@ untemplate_dir_envsubst() { unset __have_args rm -f "${__template_list}" ; unset __template_list + + return ${__ret} } -untemplate_dir_jinja() { - [ -n "${NGX_JINJA_SUFFIX:-}" ] || { log "NGX_JINJA_SUFFIX is empty" ; return 1 ; } - +expand_dir_jinja() { __template_list=$(mktemp) || return - find "$@" -follow -type f -name "*${NGX_JINJA_SUFFIX}" \ - | sort -uV > "${__template_list}" + find "$@" -follow -type f -name '*.j2' -printf '%p\0' \ + | sort -zuV > "${__template_list}" - while read -r __orig_file ; do - [ -n "${__orig_file}" ] || continue - untemplate_file_jinja "${__orig_file}" - done < "${__template_list}" - unset __orig_file + __ret=0 + if [ -s "${__template_list}" ] ; then + xargs -0r -n 1000 -a "${__template_list}" \ + j2-multi < /dev/null || __ret=1 + fi rm -f "${__template_list}" ; unset __template_list + + return ${__ret} } remap_path() { diff --git a/image-entry.d/01-defaults.envsh b/image-entry.d/01-defaults.envsh index c57fb23..cabece1 100755 --- a/image-entry.d/01-defaults.envsh +++ b/image-entry.d/01-defaults.envsh @@ -18,17 +18,6 @@ NGX_CORE_ENV="${NGX_CORE_ENV:-}" NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0) -NGX_ENVSUBST_SUFFIX="${NGX_ENVSUBST_SUFFIX:-.in}" -case "${NGX_ENVSUBST_SUFFIX}" in -.* ) ;; -* ) NGX_ENVSUBST_SUFFIX=".${NGX_ENVSUBST_SUFFIX}" ;; -esac -NGX_JINJA_SUFFIX="${NGX_JINJA_SUFFIX:-.j2}" -case "${NGX_JINJA_SUFFIX}" in -.* ) ;; -* ) NGX_JINJA_SUFFIX=".${NGX_JINJA_SUFFIX}" ;; -esac - set +a if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then diff --git a/image-entry.d/53-expand-templates.sh b/image-entry.d/53-expand-templates.sh index bb29c93..df7acd3 100755 --- a/image-entry.d/53-expand-templates.sh +++ b/image-entry.d/53-expand-templates.sh @@ -3,7 +3,15 @@ set -ef . /image-entry.d/00-common.envsh -untemplate_dir_envsubst "${merged_root}" -untemplate_dir_jinja "${merged_root}" +dirs='conf mod modules njs site snip' +[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static" + +for n in ${dirs} ; do + merged_dir="${merged_root}/$n" + [ -d "${merged_dir}" ] || continue + + expand_dir_envsubst "${merged_dir}/" + expand_dir_jinja "${merged_dir}/" +done exit 0 diff --git a/jinja2/j2-multi.py b/jinja2/j2-multi.py new file mode 100755 index 0000000..31b49a5 --- /dev/null +++ b/jinja2/j2-multi.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import os.path +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import j2common + + +def main(): + if len(sys.argv) < 2: + raise ValueError('not enough arguments (min: 1)') + + j2common.init() + + ret = 0 + for f in sys.argv[1:]: + if not j2common.render_file(f, None, False): + ret = 1 + + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/jinja2/j2-single.py b/jinja2/j2-single.py new file mode 100755 index 0000000..e76ab68 --- /dev/null +++ b/jinja2/j2-single.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import os.path +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import j2common + + +def main(): + if len(sys.argv) < 2: + raise ValueError('not enough arguments (min: 1)') + if len(sys.argv) > 3: + raise ValueError('too many arguments (max: 2)') + + if not sys.argv[1]: + raise ValueError('specify input file') + + if len(sys.argv) == 3: + if not sys.argv[2]: + raise ValueError('specify output file') + + j2common.init() + + j2common.render_file(*sys.argv[1:]) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/jinja2/j2common.py b/jinja2/j2common.py new file mode 100755 index 0000000..5a05177 --- /dev/null +++ b/jinja2/j2common.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +import importlib +import json +import os +import os.path +import sys +import yaml +import jinja2 + +ME = sys.argv[0] + +J2_MODULES_DEFAULT = 'os os.path sys netaddr psutil re wcmatch' +J2_SUFFIX = '.j2' +J2_CFG_PATHS = [ + '/angie/j2cfg', + '/etc/angie/j2cfg', +] +J2_CFG_EXTS = [ + 'yml', + 'yaml', + 'json', +] +J2_SEARCH_PATH = [ + '/etc/angie', + '/run/angie', + '/', +] +J2_EXT_LIST = [ + 'jinja2.ext.do', + 'jinja2.ext.loopcontrols', +] + +J2_MODULES = sorted(set( + os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ') +)) + +J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '') + +J2_KWARGS = {} + + +def merge_dict_from_file(filename): + if (not filename) or (filename == ''): + return False + if not os.path.exists(filename): + return False + if not os.path.isfile(filename): + print( + f'{ME}: not a file, skipping: {filename}', + file=sys.stderr) + return False + if filename.endswith('.yml') or filename.endswith('.yaml'): + with open(filename, mode='r', encoding='utf-8') as fx: + x = yaml.safe_load(fx) + J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x + return True + if filename.endswith('.json'): + with open(filename, mode='r', encoding='utf-8') as fx: + x = json.load(fx) + J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x + return True + print( + f'{ME}: non-recognized name extension: {filename}', + file=sys.stderr) + return False + + +def merge_dict_default(): + for base in J2_CFG_PATHS: + for full in [base + '.' + ext for ext in J2_CFG_EXTS]: + if merge_dict_from_file(full): + break + continue + + +def render_error(msg, fail=True) -> bool: + if fail: + raise ValueError(msg) + print(f'{ME}: {msg}', file=sys.stderr) + return False + + +def render_file(file_in, file_out=None, fail=True): + if (not file_in) or (file_in == ''): + return render_error( + 'argument "file_in" is empty', + fail) + if not os.path.exists(file_in): + return render_error( + f'file is missing: {file_in}', + fail) + if not os.path.isfile(file_in): + return render_error( + f'not a file: {file_in}', + fail) + + if file_out: + if str(file_in) == str(file_out): + return render_error( + f'unable to process template inplace: {file_in}', + fail) + f_out = file_out + else: + if not file_in.endswith(J2_SUFFIX): + return render_error( + f'input file name extension mismatch: {file_in}', + fail) + f_out = os.path.splitext(file_in)[0] + + if os.path.exists(f_out): + return render_error( + f'output file already exists: {f_out}', + fail) + + dirs = J2_SEARCH_PATH.copy() + for d in [os.path.dirname(file_in), os.getcwd()]: + if d not in dirs: + dirs.insert(0, d) + + j2_loader = jinja2.ChoiceLoader([ + jinja2.FileSystemLoader( + d, + encoding='utf-8', + followlinks=True, + ) for d in dirs + ]) + + j2_environ = jinja2.Environment( + loader=j2_loader, + extensions=J2_EXT_LIST, + ) + j2_template = j2_environ.get_template(file_in) + j2_stream = j2_template.stream(**J2_KWARGS) + j2_stream.disable_buffering() + j2_stream.dump(f_out, encoding='utf-8') + return True + + +def init(): + global J2_KWARGS + for m in J2_MODULES: + J2_KWARGS[m] = importlib.import_module(m) + J2_KWARGS['env'] = os.environ + J2_KWARGS['cfg'] = {} + + if J2_CONFIG != '': + if os.path.isfile(J2_CONFIG): + merge_dict_from_file(J2_CONFIG) + else: + print( + f'{ME}: J2_CONFIG does not exist, skipping: {J2_CONFIG}', + file=sys.stderr + ) + merge_dict_default() + else: + merge_dict_default() diff --git a/scripts/j2-multi b/scripts/j2-multi new file mode 100755 index 0000000..4970288 --- /dev/null +++ b/scripts/j2-multi @@ -0,0 +1,6 @@ +#!/bin/sh +[ "${IEP_VERBOSE}" = 0 ] || { + echo "Running ${0##*/}:" >&2 + printf ' - %s\n' "$@" >&2 +} +exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@" \ No newline at end of file diff --git a/scripts/j2-single b/scripts/j2-single new file mode 100755 index 0000000..45907e2 --- /dev/null +++ b/scripts/j2-single @@ -0,0 +1,5 @@ +#!/bin/sh +[ "${IEP_VERBOSE}" = 0 ] || { + echo "Running ${0##*/}: $*" >&2 +} +exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@" \ No newline at end of file diff --git a/scripts/jinja.py b/scripts/jinja.py deleted file mode 100755 index a592136..0000000 --- a/scripts/jinja.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -import importlib -import json -import os -import os.path -import sys - -import jinja2 -import yaml - - -J2_MODULES_DEFAULT = 'os os.path sys netaddr psutil re wcmatch' -J2_SUFFIX_DEFAULT = '.j2' -J2_CFG_PATHS = [ - '/angie/jinja', - '/etc/angie/jinja', -] -J2_CFG_EXTS = [ - 'yml', - 'yaml', - 'json', -] -J2_SEARCH_PATH = [ - '.', - '/etc/angie', -] -J2_EXT_LIST = [ - 'jinja2.ext.do', - 'jinja2.ext.loopcontrols', -] - - -J2_MODULES = sorted(set(os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' '))) - -ME = sys.argv[0] - -J2_SUFFIX = os.getenv('NGX_JINJA_SUFFIX', J2_SUFFIX_DEFAULT) -if J2_SUFFIX == '': - raise ValueError('NGX_JINJA_SUFFIX is empty') -if not J2_SUFFIX.startswith('.'): - raise ValueError('NGX_JINJA_SUFFIX does not start with dot (".")') - -J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '') -if (J2_CONFIG != '') and (not os.path.exists(J2_CONFIG)): - print(f'{ME}: config does not exist, skipping: {J2_CONFIG}', file=sys.stderr) - - -if len(sys.argv) < 2: - raise ValueError('not enough arguments (needed: 2)') -if len(sys.argv) > 3: - raise ValueError('too many arguments (needed: 2)') - -if not sys.argv[1]: - raise ValueError('specify input file') -input_file = sys.argv[1] -if not os.path.exists(input_file): - raise ValueError('input file does not exist') - -if len(sys.argv) == 3: - if not sys.argv[2]: - raise ValueError('specify output file') - output_file = sys.argv[2] -else: - output_file, ext = os.path.splitext(input_file) - if ext != J2_SUFFIX: - raise ValueError(f'input file name extension mismatch (not a "{J2_SUFFIX}")') - -if input_file == output_file: - raise ValueError('unable to process template inplace') - -kwargs = {} -for m in J2_MODULES: - kwargs[m] = importlib.import_module(m) -kwargs['env'] = os.environ -kwargs['cfg'] = {} - - -def merge_dict_from_file(filename): - if not filename: - return False - if not isinstance(filename, str): - return False - if filename == '': - return False - if not os.path.exists(filename): - return False - if not os.path.isfile(filename): - print(f'{ME}: not a file, skipping: {filename}', file=sys.stderr) - return False - if filename.endswith('.yml') or filename.endswith('.yaml'): - with open(filename, mode='r', encoding='utf-8') as fx: - x = yaml.safe_load(fx) - kwargs['cfg'] = kwargs['cfg'] | x - return True - if filename.endswith('.json'): - with open(filename, mode='r', encoding='utf-8') as fx: - x = json.load(fx) - kwargs['cfg'] = kwargs['cfg'] | x - return True - print(f'{ME}: non-recognized name extension: {filename}', file=sys.stderr) - return False - - -if (J2_CONFIG != '') and (os.path.isfile(J2_CONFIG)): - merge_dict_from_file(J2_CONFIG) -else: - for base in J2_CFG_PATHS: - for full in [ base + '.' + ext for ext in J2_CFG_EXTS ]: - if merge_dict_from_file(full): - break - continue - - -j2_loader = jinja2.FileSystemLoader(J2_SEARCH_PATH, followlinks=True) -j2_environ = jinja2.Environment(loader=j2_loader, extensions=J2_EXT_LIST) -j2_template = j2_environ.get_template(input_file) -j2_stream = j2_template.stream(**kwargs) -j2_stream.disable_buffering() -j2_stream.dump(output_file, encoding='utf-8')