From 5d3307fe57655af0c3b9c71e7c562c9d6a6d95bd Mon Sep 17 00:00:00 2001 From: Konstantin Demin Date: Sat, 20 Jul 2024 16:35:39 +0300 Subject: [PATCH] heavily rework template unrolling --- .dockerignore | 3 +- Dockerfile | 10 +- angie/conf.dist/core-preserve-env.conf.j2 | 33 +-- angie/j2cfg.dist/compress-types.yml | 15 ++ .../core-preserve-environment.txt.j2 | 7 + angie/snip.dist/brotli/buffers.conf | 5 + angie/snip.dist/brotli/gzip.conf | 1 + angie/snip.dist/brotli/types.conf.j2 | 8 + angie/snip.dist/gzip/buffers.conf | 4 + angie/snip.dist/gzip/proxied.conf | 1 + angie/snip.dist/gzip/types.conf.j2 | 8 + angie/snip.dist/gzip/vary.conf | 1 + angie/snip.dist/http-brotli.conf | 2 +- angie/snip.dist/http-brotli.modconf.j2 | 14 -- .../{http-gunzip => http-gunzip.modconf} | 0 angie/snip.dist/http-gzip.conf | 2 +- angie/snip.dist/http-gzip.modconf.j2 | 15 -- angie/snip.dist/http-zstd.conf | 2 +- angie/snip.dist/http-zstd.modconf.j2 | 12 - angie/snip.dist/mime-types.compress.j2inc | 2 - angie/snip.dist/mime-types.compress.txt | 14 -- angie/snip.dist/zstd/buffers.conf | 4 + angie/snip.dist/zstd/gzip.conf | 1 + angie/snip.dist/zstd/types.conf.j2 | 8 + image-entry.d/00-common.envsh | 14 +- image-entry.d/01-defaults.envsh | 28 ++- image-entry.d/10-core.envsh | 9 + ...re-modules.envsh => 11-core-modules.envsh} | 8 +- ...{11-core-user.envsh => 12-core-user.envsh} | 0 ...ts.envsh => 13-core-worker-defaults.envsh} | 0 image-entry.d/21-http-modules.envsh | 6 +- image-entry.d/31-mail-modules.envsh | 6 +- image-entry.d/41-stream-modules.envsh | 6 +- image-entry.d/52-merge-tree.sh | 5 +- image-entry.d/53-expand-templates.sh | 32 ++- image-entry.d/54-combine-tree.sh | 28 ++- image-entry.d/99-cleanup-env.envsh | 41 ++++ jinja2/j2-multi.py => j2cfg/j2cfg-multi.py | 9 +- jinja2/j2-single.py => j2cfg/j2cfg-single.py | 14 +- j2cfg/j2cfg/__init__.py | 225 ++++++++++++++++++ j2cfg/j2cfg/functions.py | 147 ++++++++++++ j2cfg/j2cfg/settings.py | 26 ++ jinja2/j2common.py | 159 ------------- scripts/{j2-multi => j2cfg-multi} | 2 +- scripts/{j2-single => j2cfg-single} | 2 +- 45 files changed, 619 insertions(+), 320 deletions(-) create mode 100644 angie/j2cfg.dist/compress-types.yml create mode 100644 angie/j2cfg.dist/core-preserve-environment.txt.j2 create mode 100644 angie/snip.dist/brotli/buffers.conf create mode 100644 angie/snip.dist/brotli/gzip.conf create mode 100644 angie/snip.dist/brotli/types.conf.j2 create mode 100644 angie/snip.dist/gzip/buffers.conf create mode 100644 angie/snip.dist/gzip/proxied.conf create mode 100644 angie/snip.dist/gzip/types.conf.j2 create mode 100644 angie/snip.dist/gzip/vary.conf delete mode 100644 angie/snip.dist/http-brotli.modconf.j2 rename angie/snip.dist/{http-gunzip => http-gunzip.modconf} (100%) delete mode 100644 angie/snip.dist/http-gzip.modconf.j2 delete mode 100644 angie/snip.dist/http-zstd.modconf.j2 delete mode 100644 angie/snip.dist/mime-types.compress.j2inc delete mode 100644 angie/snip.dist/mime-types.compress.txt create mode 100644 angie/snip.dist/zstd/buffers.conf create mode 100644 angie/snip.dist/zstd/gzip.conf create mode 100644 angie/snip.dist/zstd/types.conf.j2 create mode 100755 image-entry.d/10-core.envsh rename image-entry.d/{10-core-modules.envsh => 11-core-modules.envsh} (76%) rename image-entry.d/{11-core-user.envsh => 12-core-user.envsh} (100%) rename image-entry.d/{12-core-worker-defaults.envsh => 13-core-worker-defaults.envsh} (100%) create mode 100755 image-entry.d/99-cleanup-env.envsh rename jinja2/j2-multi.py => j2cfg/j2cfg-multi.py (63%) rename jinja2/j2-single.py => j2cfg/j2cfg-single.py (59%) create mode 100644 j2cfg/j2cfg/__init__.py create mode 100644 j2cfg/j2cfg/functions.py create mode 100644 j2cfg/j2cfg/settings.py delete mode 100755 jinja2/j2common.py rename scripts/{j2-multi => j2cfg-multi} (76%) rename scripts/{j2-single => j2cfg-single} (75%) diff --git a/.dockerignore b/.dockerignore index 1df2d86..4d5ab1c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -jinja2/__pycache__ +j2cfg/__pycache__ +j2cfg/j2cfg/__pycache__ diff --git a/Dockerfile b/Dockerfile index 7306f12..eeb950e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,13 +26,13 @@ SHELL [ "/bin/sh", "-ec" ] COPY /scripts/* /usr/local/sbin/ COPY /extra-scripts/* /usr/local/sbin/ -COPY /jinja2/ /usr/local/lib/jinja2/ +COPY /j2cfg/ /usr/local/lib/j2cfg/ ENV PYTHONDONTWRITEBYTECODE='' ## Python cache preseed -RUN python3 -m compileall -q -j 2 /usr/local/lib/jinja2/ +RUN python3 -m compileall -q -j 2 /usr/local/lib/j2cfg/ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \ @@ -51,7 +51,7 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \ ## Python cache warmup RUN echo > /tmp/f.j2 ; \ - j2-single /tmp/f.j2 ; \ + j2cfg-single /tmp/f.j2 ; \ rm -f /tmp/f /tmp/f.j2 ## Python cache adjustments @@ -76,7 +76,7 @@ COPY --from=certs /usr/local/share/ca-certificates/ /usr/local/share/ca-certif COPY --from=pycache /usr/local/lib/ /usr/local/lib/ ## already copied by statement above -# COPY /jinja2/ /usr/local/lib/jinja2/ +# COPY /j2cfg/ /usr/local/lib/j2cfg/ ENV ANGIE_MODULES_DIR=/usr/lib/angie/modules @@ -141,7 +141,7 @@ RUN install -d -o angie -g angie -m 03777 /angie /run/angie ; \ ln -sv /run/angie/lock lock.d ; \ ln -sv ${ANGIE_MODULES_DIR} modules.dist ; \ ## hyper-modular paths: - data='conf mod modules njs site snip static' ; \ + data='conf j2cfg mod modules njs site snip static' ; \ vardata='cache lib log' ; \ for n in ${data} ; do \ for d in "$n" "$n.dist" ; do \ diff --git a/angie/conf.dist/core-preserve-env.conf.j2 b/angie/conf.dist/core-preserve-env.conf.j2 index f4ff5e5..8b74b09 100644 --- a/angie/conf.dist/core-preserve-env.conf.j2 +++ b/angie/conf.dist/core-preserve-env.conf.j2 @@ -1,33 +1,18 @@ {#- prologue -#} -{%- set penv = [] -%} -{%- if cfg.preserve_env -%} - {%- set penv = cfg.preserve_env -%} - {%- if penv is string -%} - {%- set penv = [penv] -%} - {%- elif penv is iterable -%} - {#- {%- set penv = penv -%} -#} - {%- else -%} - {%- set penv = [penv|string()] -%} - {%- endif -%} -{%- endif -%} -{%- set have = namespace() -%} -{%- set have.tz = false -%} -{%- set have.malloc_arena = false -%} -{#- scan -#} -{%- for v in penv -%} - {%- set have.tz = have.tz or re.match('TZ(=|$)', v|string()) -%} - {%- set have.malloc_arena = have.malloc_arena or re.match('MALLOC_ARENA_MAX(=|$)', v|string()) -%} -{%- endfor -%} +{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%} +{%- set have_tz = preserve_env|is_str_list_re_match('TZ(=|$)') -%} +{%- set have_malloc_arena = preserve_env|is_str_list_re_match('MALLOC_ARENA_MAX(=|$)') -%} {#- main part -#} -{%- if not have.tz -%} +{%- if not have_tz -%} env TZ; {% endif %} -{%- if not have.malloc_arena -%} +{%- if not have_malloc_arena -%} env MALLOC_ARENA_MAX; {% endif %} -{%- for v in penv -%} - {%- if re.search("(\"|'|\\s)", v|string()) %} -env {{ (v|string()).__repr__() }}; +{%- for v in preserve_env -%} + {%- if re.search("(\"|'|\\s)", v) %} +{#- TODO: investigate corrent escape behavior for Angie/nginx -#} +env {{ v.__repr__() }}; {%- else %} env {{ v }}; {%- endif %} diff --git a/angie/j2cfg.dist/compress-types.yml b/angie/j2cfg.dist/compress-types.yml new file mode 100644 index 0000000..bdfacf4 --- /dev/null +++ b/angie/j2cfg.dist/compress-types.yml @@ -0,0 +1,15 @@ +compress_types: +- application/atom+xml +- application/javascript +- application/json +- application/vnd.api+json +- application/rss+xml +- application/x-javascript +- application/xhtml+xml +- application/xml +- image/svg+xml +- image/x-icon +- text/css +- text/javascript +- text/plain +- text/xml \ No newline at end of file diff --git a/angie/j2cfg.dist/core-preserve-environment.txt.j2 b/angie/j2cfg.dist/core-preserve-environment.txt.j2 new file mode 100644 index 0000000..6aa02b6 --- /dev/null +++ b/angie/j2cfg.dist/core-preserve-environment.txt.j2 @@ -0,0 +1,7 @@ +{#- prologue -#} +{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%} +{%- set preserve_vars = preserve_env|str_list_re_fullmatch('[^=]+') -%} +{#- main part -#} +{% for v in preserve_vars -%} +{{ v }} +{% endfor -%} diff --git a/angie/snip.dist/brotli/buffers.conf b/angie/snip.dist/brotli/buffers.conf new file mode 100644 index 0000000..5cf3b78 --- /dev/null +++ b/angie/snip.dist/brotli/buffers.conf @@ -0,0 +1,5 @@ +brotli_comp_level 5; # default: 6 +brotli_window 64k; # default: 512k + +brotli_min_length 1024; +brotli_buffers 32 16k; \ No newline at end of file diff --git a/angie/snip.dist/brotli/gzip.conf b/angie/snip.dist/brotli/gzip.conf new file mode 100644 index 0000000..76f9958 --- /dev/null +++ b/angie/snip.dist/brotli/gzip.conf @@ -0,0 +1 @@ +include snip.d/gzip/vary.conf; \ No newline at end of file diff --git a/angie/snip.dist/brotli/types.conf.j2 b/angie/snip.dist/brotli/types.conf.j2 new file mode 100644 index 0000000..bf09900 --- /dev/null +++ b/angie/snip.dist/brotli/types.conf.j2 @@ -0,0 +1,8 @@ +{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%} +{%- if mime_types %} +brotli_types +{%- for t in mime_types %} + {{ t }} +{%- endfor %} +; +{%- endif %} \ No newline at end of file diff --git a/angie/snip.dist/gzip/buffers.conf b/angie/snip.dist/gzip/buffers.conf new file mode 100644 index 0000000..8a13413 --- /dev/null +++ b/angie/snip.dist/gzip/buffers.conf @@ -0,0 +1,4 @@ +gzip_comp_level 2; # default: 1 + +gzip_min_length 1024; +gzip_buffers 32 16k; \ No newline at end of file diff --git a/angie/snip.dist/gzip/proxied.conf b/angie/snip.dist/gzip/proxied.conf new file mode 100644 index 0000000..989c9d7 --- /dev/null +++ b/angie/snip.dist/gzip/proxied.conf @@ -0,0 +1 @@ +gzip_proxied any; \ No newline at end of file diff --git a/angie/snip.dist/gzip/types.conf.j2 b/angie/snip.dist/gzip/types.conf.j2 new file mode 100644 index 0000000..34f084d --- /dev/null +++ b/angie/snip.dist/gzip/types.conf.j2 @@ -0,0 +1,8 @@ +{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%} +{%- if mime_types %} +gzip_types +{%- for t in mime_types %} + {{ t }} +{%- endfor %} +; +{%- endif %} \ No newline at end of file diff --git a/angie/snip.dist/gzip/vary.conf b/angie/snip.dist/gzip/vary.conf new file mode 100644 index 0000000..d157ade --- /dev/null +++ b/angie/snip.dist/gzip/vary.conf @@ -0,0 +1 @@ +gzip_vary on; \ No newline at end of file diff --git a/angie/snip.dist/http-brotli.conf b/angie/snip.dist/http-brotli.conf index 0bd698f..e06238a 100644 --- a/angie/snip.dist/http-brotli.conf +++ b/angie/snip.dist/http-brotli.conf @@ -1,2 +1,2 @@ -include snip.d/http-brotli.modconf; +include snip.d/brotli/*.conf; brotli on; \ No newline at end of file diff --git a/angie/snip.dist/http-brotli.modconf.j2 b/angie/snip.dist/http-brotli.modconf.j2 deleted file mode 100644 index 5bdf5a7..0000000 --- a/angie/snip.dist/http-brotli.modconf.j2 +++ /dev/null @@ -1,14 +0,0 @@ -{%- from 'mime-types.compress.j2inc' import mime_types with context -%} - -## default is 6 -brotli_comp_level 5; -## default is 512k -brotli_window 64k; - -brotli_min_length 1024; -brotli_buffers 32 16k; - -brotli_types -## sourced from mime-types.compress.txt - {{ mime_types | indent(4) }} -; \ No newline at end of file diff --git a/angie/snip.dist/http-gunzip b/angie/snip.dist/http-gunzip.modconf similarity index 100% rename from angie/snip.dist/http-gunzip rename to angie/snip.dist/http-gunzip.modconf diff --git a/angie/snip.dist/http-gzip.conf b/angie/snip.dist/http-gzip.conf index 0b070c2..ffd80c6 100644 --- a/angie/snip.dist/http-gzip.conf +++ b/angie/snip.dist/http-gzip.conf @@ -1,2 +1,2 @@ -include snip.d/http-gzip.modconf; +include snip.d/gzip/*.conf; gzip on; \ No newline at end of file diff --git a/angie/snip.dist/http-gzip.modconf.j2 b/angie/snip.dist/http-gzip.modconf.j2 deleted file mode 100644 index 42db9f1..0000000 --- a/angie/snip.dist/http-gzip.modconf.j2 +++ /dev/null @@ -1,15 +0,0 @@ -{%- from 'mime-types.compress.j2inc' import mime_types with context -%} - -## default is 1 -gzip_comp_level 2; - -gzip_min_length 1024; -gzip_buffers 32 16k; - -gzip_vary on; -gzip_proxied any; - -gzip_types -## sourced from mime-types.compress.txt - {{ mime_types | indent(4) }} -; \ No newline at end of file diff --git a/angie/snip.dist/http-zstd.conf b/angie/snip.dist/http-zstd.conf index 7dbb8c2..6e8540d 100644 --- a/angie/snip.dist/http-zstd.conf +++ b/angie/snip.dist/http-zstd.conf @@ -1,2 +1,2 @@ -include snip.d/http-zstd.modconf; +include snip.d/zstd/*.conf; zstd on; \ No newline at end of file diff --git a/angie/snip.dist/http-zstd.modconf.j2 b/angie/snip.dist/http-zstd.modconf.j2 deleted file mode 100644 index fb4d984..0000000 --- a/angie/snip.dist/http-zstd.modconf.j2 +++ /dev/null @@ -1,12 +0,0 @@ -{%- from 'mime-types.compress.j2inc' import mime_types with context -%} - -## default is 1 -zstd_comp_level 2; - -zstd_min_length 1024; -zstd_buffers 32 16k; - -zstd_types -## sourced from mime-types.compress.txt - {{ mime_types | indent(4) }} -; \ No newline at end of file diff --git a/angie/snip.dist/mime-types.compress.j2inc b/angie/snip.dist/mime-types.compress.j2inc deleted file mode 100644 index ed2f4d0..0000000 --- a/angie/snip.dist/mime-types.compress.j2inc +++ /dev/null @@ -1,2 +0,0 @@ -{%- set mime_file = pathlib.Path(os.path.join(os.getenv('NGX_MERGED_ROOT'), 'snip/mime-types.compress.txt')) -%} -{%- set mime_types = mime_file.read_text() -%} diff --git a/angie/snip.dist/mime-types.compress.txt b/angie/snip.dist/mime-types.compress.txt deleted file mode 100644 index d2e12ab..0000000 --- a/angie/snip.dist/mime-types.compress.txt +++ /dev/null @@ -1,14 +0,0 @@ -application/atom+xml -application/javascript -application/json -application/vnd.api+json -application/rss+xml -application/x-javascript -application/xhtml+xml -application/xml -image/svg+xml -image/x-icon -text/css -text/javascript -text/plain -text/xml \ No newline at end of file diff --git a/angie/snip.dist/zstd/buffers.conf b/angie/snip.dist/zstd/buffers.conf new file mode 100644 index 0000000..c310cc5 --- /dev/null +++ b/angie/snip.dist/zstd/buffers.conf @@ -0,0 +1,4 @@ +zstd_comp_level 2; # default: 1 + +zstd_min_length 1024; +zstd_buffers 32 16k; \ No newline at end of file diff --git a/angie/snip.dist/zstd/gzip.conf b/angie/snip.dist/zstd/gzip.conf new file mode 100644 index 0000000..76f9958 --- /dev/null +++ b/angie/snip.dist/zstd/gzip.conf @@ -0,0 +1 @@ +include snip.d/gzip/vary.conf; \ No newline at end of file diff --git a/angie/snip.dist/zstd/types.conf.j2 b/angie/snip.dist/zstd/types.conf.j2 new file mode 100644 index 0000000..32934aa --- /dev/null +++ b/angie/snip.dist/zstd/types.conf.j2 @@ -0,0 +1,8 @@ +{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%} +{%- if mime_types %} +zstd_types +{%- for t in mime_types %} + {{ t }} +{%- endfor %} +; +{%- endif %} \ No newline at end of file diff --git a/image-entry.d/00-common.envsh b/image-entry.d/00-common.envsh index 7dea860..0aaa51b 100644 --- a/image-entry.d/00-common.envsh +++ b/image-entry.d/00-common.envsh @@ -82,7 +82,7 @@ untemplate_path() { "${volume_root}"/* | /etc/angie/run/* ) strip_suffix "$1" "$2" ;; - /etc/angie/conf.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* ) + /etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* ) strip_suffix "$1" "$2" ;; /etc/angie/static.d/* ) @@ -151,8 +151,8 @@ expand_file_envsubst() { return ${__r} } -expand_file_jinja() { - j2-single "$@" || return $? +expand_file_j2cfg() { + j2cfg-single "$@" || return $? } expand_dir_envsubst() { @@ -187,7 +187,7 @@ expand_dir_envsubst() { return ${__ret} } -expand_dir_jinja() { +expand_dir_j2cfg() { __template_list=$(mktemp) || return find "$@" -follow -type f -name '*.j2' -printf '%p\0' \ @@ -196,7 +196,7 @@ expand_dir_jinja() { __ret=0 if [ -s "${__template_list}" ] ; then xargs -0r -n 1000 -a "${__template_list}" \ - j2-multi < /dev/null || __ret=1 + j2cfg-multi < /dev/null || __ret=1 fi rm -f "${__template_list}" ; unset __template_list @@ -212,6 +212,10 @@ remap_path() { /etc/angie/conf.dist/* ) echo "${2:-/etc/angie/conf.d}${1#/etc/angie/conf.dist}" ;; /etc/angie/conf/* ) echo "${2:-/etc/angie/conf.d}${1#/etc/angie/conf}" ;; /angie/conf/* ) echo "${2:-/etc/angie/conf.d}${1#/angie/conf}" ;; + ## j2cfg + /etc/angie/j2cfg.dist/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/etc/angie/j2cfg.dist}" ;; + /etc/angie/j2cfg/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/etc/angie/j2cfg}" ;; + /angie/j2cfg/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/angie/j2cfg}" ;; ## mod /etc/angie/mod.dist/* ) echo "${2:-/etc/angie/mod.d}${1#/etc/angie/mod.dist}" ;; /etc/angie/mod/* ) echo "${2:-/etc/angie/mod.d}${1#/etc/angie/mod}" ;; diff --git a/image-entry.d/01-defaults.envsh b/image-entry.d/01-defaults.envsh index 900a129..e51239f 100755 --- a/image-entry.d/01-defaults.envsh +++ b/image-entry.d/01-defaults.envsh @@ -3,19 +3,12 @@ ## NB: NGX_DEBUG is set via image build script set -a +NGX_STRICT_LOAD=$(gobool_to_int "${NGX_STRICT_LOAD:-1}" 1) +NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0) NGX_HTTP=$(gobool_to_int "${NGX_HTTP:-1}" 1) NGX_MAIL=$(gobool_to_int "${NGX_MAIL:-0}" 0) NGX_STREAM=$(gobool_to_int "${NGX_STREAM:-0}" 0) - -NGX_STRICT_LOAD=$(gobool_to_int "${NGX_STRICT_LOAD:-1}" 1) - -NGX_CORE_MODULES="${NGX_CORE_MODULES:-}" -NGX_CORE_EVENTS_SNIPPETS="${NGX_CORE_EVENTS_SNIPPETS:-}" -NGX_CORE_SNIPPETS="${NGX_CORE_SNIPPETS:-}" - -NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0) - set +a if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then @@ -24,3 +17,20 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then log_always 'Angie is almost completely TURNED OFF' log_always '======================================' fi + +unset default_dirs_merge default_dirs_link +default_dirs_merge='conf j2cfg mod modules njs site snip' +default_dirs_link='' + +if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then + NGX_DIRS_MERGE="${NGX_DIRS_MERGE:-} static" +else + NGX_DIRS_LINK="${NGX_DIRS_LINK:-} static" +fi + +set -a +NGX_DIRS_MERGE=$(sort_dedup_list "${default_dirs_merge} ${NGX_DIRS_MERGE:-}") +NGX_DIRS_LINK=$(sort_dedup_list "${default_dirs_link} ${NGX_DIRS_LINK:-}") +set +a + +unset default_dirs_merge default_dirs_link diff --git a/image-entry.d/10-core.envsh b/image-entry.d/10-core.envsh new file mode 100755 index 0000000..a4efb06 --- /dev/null +++ b/image-entry.d/10-core.envsh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -a + +NGX_CORE_MODULES="${NGX_CORE_MODULES:-}" +NGX_CORE_EVENTS_SNIPPETS="${NGX_CORE_EVENTS_SNIPPETS:-}" +NGX_CORE_SNIPPETS="${NGX_CORE_SNIPPETS:-}" + +set +a diff --git a/image-entry.d/10-core-modules.envsh b/image-entry.d/11-core-modules.envsh similarity index 76% rename from image-entry.d/10-core-modules.envsh rename to image-entry.d/11-core-modules.envsh index ad083fa..81cd1f9 100755 --- a/image-entry.d/10-core-modules.envsh +++ b/image-entry.d/11-core-modules.envsh @@ -11,7 +11,7 @@ for i in ${NGX_CORE_MODULES:-} ; do if is_builtin_module core "$i" ; then log "$i is builtin module, moving to snippets" - core_snippets="${core_snippets}${core_snippets:+ }$i" + core_snippets="${core_snippets} $i" continue fi @@ -27,12 +27,10 @@ for i in ${NGX_CORE_MODULES:-} ; do done unset i -## sort and remove duplicates -core_snippets=$(sort_dedup_list "${core_snippets}") - set -a NGX_CORE_MODULES="${core_modules}" -NGX_CORE_SNIPPETS="${core_snippets}" +NGX_CORE_SNIPPETS=$(sort_dedup_list "${core_snippets}") +NGX_CORE_EVENTS_SNIPPETS=$(sort_dedup_list "${NGX_CORE_EVENTS_SNIPPETS}") set +a unset core_modules core_snippets diff --git a/image-entry.d/11-core-user.envsh b/image-entry.d/12-core-user.envsh similarity index 100% rename from image-entry.d/11-core-user.envsh rename to image-entry.d/12-core-user.envsh diff --git a/image-entry.d/12-core-worker-defaults.envsh b/image-entry.d/13-core-worker-defaults.envsh similarity index 100% rename from image-entry.d/12-core-worker-defaults.envsh rename to image-entry.d/13-core-worker-defaults.envsh diff --git a/image-entry.d/21-http-modules.envsh b/image-entry.d/21-http-modules.envsh index 6e2f22e..e085766 100755 --- a/image-entry.d/21-http-modules.envsh +++ b/image-entry.d/21-http-modules.envsh @@ -21,7 +21,7 @@ if [ "${NGX_HTTP}" = 1 ] ; then if is_builtin_module http "$i" ; then log "$i is builtin module, moving to snippets" - http_snippets="${http_snippets}${http_snippets:+ }$i" + http_snippets="${http_snippets} $i" continue fi @@ -37,11 +37,9 @@ if [ "${NGX_HTTP}" = 1 ] ; then done unset i - http_snippets=$(sort_dedup_list "${http_snippets}") - set -a NGX_HTTP_MODULES="${http_modules}" - NGX_HTTP_SNIPPETS="${http_snippets}" + NGX_HTTP_SNIPPETS=$(sort_dedup_list "${http_snippets}") set +a unset http_modules http_snippets diff --git a/image-entry.d/31-mail-modules.envsh b/image-entry.d/31-mail-modules.envsh index ca554e6..84bdc27 100755 --- a/image-entry.d/31-mail-modules.envsh +++ b/image-entry.d/31-mail-modules.envsh @@ -12,7 +12,7 @@ if [ "${NGX_MAIL}" = 1 ] ; then if is_builtin_module mail "$i" ; then log "$i is builtin module, moving to snippets" - mail_snippets="${mail_snippets}${mail_snippets:+ }$i" + mail_snippets="${mail_snippets} $i" continue fi @@ -28,11 +28,9 @@ if [ "${NGX_MAIL}" = 1 ] ; then done unset i - mail_snippets=$(sort_dedup_list "${mail_snippets}") - set -a NGX_MAIL_MODULES="${mail_modules}" - NGX_MAIL_SNIPPETS="${mail_snippets}" + NGX_MAIL_SNIPPETS=$(sort_dedup_list "${mail_snippets}") set +a unset mail_modules mail_snippets diff --git a/image-entry.d/41-stream-modules.envsh b/image-entry.d/41-stream-modules.envsh index bebd540..076d148 100755 --- a/image-entry.d/41-stream-modules.envsh +++ b/image-entry.d/41-stream-modules.envsh @@ -12,7 +12,7 @@ if [ "${NGX_STREAM}" = 1 ] ; then if is_builtin_module stream "$i" ; then log "$i is builtin module, moving to snippets" - stream_snippets="${stream_snippets}${stream_snippets:+ }$i" + stream_snippets="${stream_snippets} $i" continue fi @@ -28,11 +28,9 @@ if [ "${NGX_STREAM}" = 1 ] ; then done unset i - stream_snippets=$(sort_dedup_list "${stream_snippets}") - set -a NGX_STREAM_MODULES="${stream_modules}" - NGX_STREAM_SNIPPETS="${stream_snippets}" + NGX_STREAM_SNIPPETS=$(sort_dedup_list "${stream_snippets}") set +a unset stream_modules stream_snippets diff --git a/image-entry.d/52-merge-tree.sh b/image-entry.d/52-merge-tree.sh index e6f88f2..12a2085 100755 --- a/image-entry.d/52-merge-tree.sh +++ b/image-entry.d/52-merge-tree.sh @@ -5,10 +5,9 @@ set -ef [ -d "${merged_root}" ] || install -d "${merged_root}" -dirs='conf mod modules njs site snip' -[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static" +for n in ${NGX_DIRS_MERGE} ; do + [ -n "$n" ] || continue -for n in ${dirs} ; do merged_dir="${merged_root}/$n" while read -r old_path ; do [ -n "${old_path}" ] || continue diff --git a/image-entry.d/53-expand-templates.sh b/image-entry.d/53-expand-templates.sh index 7110501..b0f881b 100755 --- a/image-entry.d/53-expand-templates.sh +++ b/image-entry.d/53-expand-templates.sh @@ -5,31 +5,41 @@ set -f [ "${NGX_STRICT_LOAD}" = 0 ] || set -e -export NGX_MERGED_ROOT="${merged_root}" +cd "${merged_root}/" +expand_error_delim() { + IEP_TRACE=0 log_always ' ----------------------------------- ' +} expand_error() { - [ "${expand_error_seen:-}" = 1 ] || log_always 'template expansion has failed' + [ "${expand_error_seen:-}" != 1 ] || return expand_error_seen=1 + expand_error_delim + log_always 'template expansion has failed' if [ "${NGX_STRICT_LOAD}" = 1 ] ; then - t=10 + t=15 log_always "injecting delay for $t seconds" + expand_error_delim sleep $t exit 1 fi + expand_error_delim } -dirs='conf mod modules njs site snip' -[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static" - merge_dirs= -for n in ${dirs} ; do - merged_dir="${merged_root}/$n" - [ -d "${merged_dir}" ] || continue +for n in ${NGX_DIRS_MERGE} ; do + [ -n "$n" ] || continue + [ -d "$n" ] || continue - merge_dirs="${merge_dirs} ${merged_dir}/" + merge_dirs="${merge_dirs} $n/" done expand_dir_envsubst ${merge_dirs} || expand_error -expand_dir_jinja ${merge_dirs} || expand_error + +set -a +J2CFG_PATH="${merged_root}/j2cfg" +J2CFG_SEARCH_PATH="${merged_root}" +set -a + +expand_dir_j2cfg ${merge_dirs} || expand_error exit 0 diff --git a/image-entry.d/54-combine-tree.sh b/image-entry.d/54-combine-tree.sh index 2f19513..03e2b74 100755 --- a/image-entry.d/54-combine-tree.sh +++ b/image-entry.d/54-combine-tree.sh @@ -30,7 +30,7 @@ load_error() { load_error_delim log_always 'tree combine has failed' if [ "${NGX_STRICT_LOAD}" = 1 ] ; then - t=10 + t=15 log_always "injecting delay for $t seconds" load_error_delim sleep $t @@ -53,13 +53,11 @@ done ## provide same symlinks as upstream (both Angie and nginx) docker images do d="${target_root}/log" -[ -e "$d/access.log" ] || ln_s /dev/stdout "$d/access.log" -[ -e "$d/error.log" ] || ln_s /dev/stderr "$d/error.log" +[ -e "$d/access.log" ] || ln_s /dev/stdout "$d/access.log" || load_error +[ -e "$d/error.log" ] || ln_s /dev/stderr "$d/error.log" || load_error ## NB: if any error occurs above then configuration is merely empty and/or broken -dirs='conf mod modules njs site snip' -[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static" while read -r old_path ; do [ -n "${old_path}" ] || continue @@ -73,7 +71,9 @@ while read -r old_path ; do done <<-EOF $( set +e - for n in ${dirs} ; do + for n in ${NGX_DIRS_MERGE} ; do + [ -n "$n" ] || continue + [ -d "${merged_root}/$n" ] || continue find "${merged_root}/$n/" ! -type d done \ @@ -81,17 +81,23 @@ $( -e "^${merged_root}/(mod|snip)/.+\.load\$" \ -e "^${merged_root}/mod/[^/]+\.preseed\$" \ | sort -V - set -e ) EOF -if [ "${NGX_PROCESS_STATIC}" = 0 ] ; then - for d in /angie/static /etc/angie/static /etc/angie/static.dist ; do +for n in ${NGX_DIRS_LINK} ; do + [ -n "$n" ] || continue + + if [ -e "${target_root}/$n" ] ; then continue ; fi + for d in "/angie/$n" "/etc/angie/$n" "/etc/angie/$n.dist" ; do [ -d "$d" ] || continue - ln_s "$d" "${target_root}/static" + ln_s "$d" "${target_root}/$n" break done -fi + + [ -d "${target_root}/$n" ] || { + log "missing required directory: ${target_root}/$n" + } +done ## Angie modules are loaded in [strict] order! combine_modules() { diff --git a/image-entry.d/99-cleanup-env.envsh b/image-entry.d/99-cleanup-env.envsh new file mode 100755 index 0000000..cd56f18 --- /dev/null +++ b/image-entry.d/99-cleanup-env.envsh @@ -0,0 +1,41 @@ +#!/bin/sh + +__set="$-" +set +e +if [ "${IEP_TRACE}" = 1 ] ; then + log_always "NOT going to unset following variables:" + sed -E '/^./s,^,- ,' >&2 +else + unset __env + while read -r __env ; do + [ -n "${__env}" ] || continue + + case "${__env}" in + *\'* ) + log "skipping variable (malformed): ${__env}" >&2 + continue + ;; + esac + + log "unsetting variable: ${__env}" + unset "${__env}" + done + unset __env +fi <<-EOF +$( + cat /proc/self/environ \ + | sed -zEn '/^([^=]+).*$/s//\1/p' \ + | xargs -0r printf '%q\n' \ + | { + f="${target_root}/j2cfg/core-preserve-environment.txt" + [ -s "$f" ] || exec cat + grep -Fxv -f "$f" + } \ + | grep -E \ + -e '^(NGX|PYTHON)' \ + | sort -uV +) +EOF + +[ -z "${__set}" ] || set -"${__set}" +unset __set diff --git a/jinja2/j2-multi.py b/j2cfg/j2cfg-multi.py similarity index 63% rename from jinja2/j2-multi.py rename to j2cfg/j2cfg-multi.py index 31b49a5..b86050d 100755 --- a/jinja2/j2-multi.py +++ b/j2cfg/j2cfg-multi.py @@ -3,19 +3,18 @@ import os.path import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import j2common - def main(): if len(sys.argv) < 2: raise ValueError('not enough arguments (min: 1)') - j2common.init() + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import j2cfg + r = j2cfg.J2cfg(strict=False) ret = 0 for f in sys.argv[1:]: - if not j2common.render_file(f, None, False): + if not r.render_file(f, None): ret = 1 sys.exit(ret) diff --git a/jinja2/j2-single.py b/j2cfg/j2cfg-single.py similarity index 59% rename from jinja2/j2-single.py rename to j2cfg/j2cfg-single.py index e76ab68..09f4e02 100755 --- a/jinja2/j2-single.py +++ b/j2cfg/j2cfg-single.py @@ -3,9 +3,6 @@ import os.path import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import j2common - def main(): if len(sys.argv) < 2: @@ -16,13 +13,14 @@ def main(): if not sys.argv[1]: raise ValueError('specify input file') - if len(sys.argv) == 3: - if not sys.argv[2]: - raise ValueError('specify output file') + if (len(sys.argv) == 3) and (not sys.argv[2]): + raise ValueError('specify output file') - j2common.init() + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import j2cfg - j2common.render_file(*sys.argv[1:]) + r = j2cfg.J2cfg() + r.render_file(*sys.argv[1:]) sys.exit(0) diff --git a/j2cfg/j2cfg/__init__.py b/j2cfg/j2cfg/__init__.py new file mode 100644 index 0000000..0d7d1da --- /dev/null +++ b/j2cfg/j2cfg/__init__.py @@ -0,0 +1,225 @@ +import importlib +import json +import os +import os.path +import sys + +import jinja2 +import wcmatch.wcmatch +import yaml + +from .functions import * +from .settings import * + + +J2CFG_CONFIG_EXT = ['yml', 'yaml', 'json'] + + +class J2cfg: + def __init__(self, strict=True, config=None, config_path=None, + modules=None, search_path=None, template_suffix=None): + + self.strict = strict + if not isinstance(self.strict, bool): + self.strict = True + + self.config_file = config or os.getenv('J2CFG_CONFIG') + if self.config_file is not None: + self.config_file = str(self.config_file) + + self.config_path = config_path + if self.config_path is None: + self.config_path = os.getenv('J2CFG_PATH') + if self.config_path is not None: + self.config_path = str_split_to_list(self.config_path, ':') + if self.config_path is None: + self.config_path = J2CFG_PATH.copy() + else: + self.config_path = any_to_str_list(self.config_path) + self.config_path = uniq_str_list(self.config_path) + + self.search_path = search_path + if self.search_path is None: + self.search_path = os.getenv('J2CFG_SEARCH_PATH') + if self.search_path is not None: + self.search_path = str_split_to_list(self.search_path, ':') + if self.search_path is None: + self.search_path = self.config_path.copy() + else: + self.search_path = any_to_str_list(self.search_path) + self.search_path = uniq_str_list(self.search_path) + # RFC: should we use the current working directory early? + for d in [os.getcwd()]: + if d not in self.search_path: + self.search_path.insert(0, d) + + self.modules = modules or os.getenv('J2CFG_MODULES') + if self.modules is None: + self.modules = J2CFG_PYTHON_MODULES.copy() + else: + if isinstance(self.modules, str): + self.modules = str_split_to_list(self.modules) + else: + self.modules = any_to_str_list(self.modules) + self.modules = uniq_str_list(self.modules) + + self.template_suffix = template_suffix or os.getenv('J2CFG_SUFFIX') + if self.template_suffix is None: + self.template_suffix = J2CFG_TEMPLATE_EXT + else: + self.template_suffix = str(self.template_suffix) + if self.template_suffix == '': + self.template_suffix = J2CFG_TEMPLATE_EXT + if not self.template_suffix.startswith('.'): + self.template_suffix = '.' + self.template_suffix + + self.kwargs = { + 'env': os.environ, + 'j2cfg': {} + } + for m in self.modules: + if m in self.kwargs: + print(f'J2cfg: kwargs already has {m} key', + file=sys.stderr) + continue + self.kwargs[m] = importlib.import_module(m) + + def merge_dict_from_file(filename): + if filename is None: + return False + f = str(filename) + if f == '': + return False + if not os.path.exists(f): + return False + if not os.path.isfile(f): + print( + f'J2cfg: not a file, skipping: {filename}', + file=sys.stderr) + return False + + if f.endswith('.yml') or f.endswith('.yaml'): + with open(f, mode='r', encoding='utf-8') as fx: + x = yaml.safe_load(fx) + self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x + return True + + if f.endswith('.json'): + with open(f, mode='r', encoding='utf-8') as fx: + x = json.load(fx) + self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x + return True + + print( + f'J2cfg: non-recognized name extension: {filename}', + file=sys.stderr) + return False + + def merge_dict_default(): + search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT]) + search_flags = wcmatch.wcmatch.RECURSIVE | wcmatch.wcmatch.SYMLINKS + + for d in self.config_path: + if not os.path.isdir(d): + continue + for f in wcmatch.wcmatch.WcMatch(d, search_pattern, + flags=search_flags).imatch(): + merge_dict_from_file(f) + + if self.config_file is None: + merge_dict_default() + else: + if os.path.isfile(self.config_file): + merge_dict_from_file(self.config_file) + else: + print( + 'J2cfg: J2cfg config file does not exist, skipping: ' + + f'{self.config_file}', + file=sys.stderr + ) + + self.j2fs_loaders = { + d: jinja2.FileSystemLoader( + d, encoding='utf-8', followlinks=True, + ) for d in self.search_path + } + self.j2env = jinja2.Environment( + extensions=J2CFG_JINJA_EXTENSIONS, + loader=jinja2.ChoiceLoader([ + self.j2fs_loaders[d] for d in self.search_path + ]), + ) + + def init_env(e: jinja2.Environment): + for s in J2CFG_FILTERS: + n = s.__name__ + if n in e.filters: + print(f'J2cfg: filters already has {n} key', + file=sys.stderr) + continue + e.filters[n] = s + + init_env(self.j2env) + + def ensure_fs_loader_for(self, directory: str): + if directory in self.j2fs_loaders: + return + self.j2fs_loaders[directory] = jinja2.FileSystemLoader( + directory, encoding='utf-8', followlinks=True, + ) + + def render_file(self, file_in, file_out=None) -> bool: + + def render_error(msg) -> bool: + if self.strict: + raise ValueError(msg) + print(f'J2cfg: {msg}', file=sys.stderr) + return False + + if file_in is None: + return render_error( + 'argument "file_in" is None') + f_in = str(file_in) + if f_in == '': + return render_error( + 'argument "file_in" is empty') + if not os.path.exists(f_in): + return render_error( + f'file is missing: {file_in}') + if not os.path.isfile(f_in): + return render_error( + f'not a file: {file_in}') + + f_out = file_out + if f_out is None: + if not f_in.endswith(self.template_suffix): + return render_error( + f'input file name extension mismatch: {file_in}') + f_out = os.path.splitext(f_in)[0] + + dirs = self.search_path.copy() + for d in [os.getcwd(), os.path.dirname(f_in)]: + if d in dirs: + continue + self.ensure_fs_loader_for(d) + dirs.insert(0, d) + + j2_environ = self.j2env.overlay(loader=jinja2.ChoiceLoader([ + self.j2fs_loaders[d] for d in dirs + ])) + j2_template = j2_environ.get_template(f_in) + rendered = j2_template.render(**self.kwargs) + + if os.path.lexists(f_out): + if os.path.islink(f_out) or (not os.path.isfile(f_out)): + return render_error( + f'output file is not safely writable: {f_out}') + if os.path.exists(f_out): + if os.path.samefile(f_in, f_out): + return render_error( + f'unable to process template inplace: {file_in}') + + with open(f_out, mode='w', encoding='utf-8') as f: + f.write(rendered) + + return True diff --git a/j2cfg/j2cfg/functions.py b/j2cfg/j2cfg/functions.py new file mode 100644 index 0000000..0520b7c --- /dev/null +++ b/j2cfg/j2cfg/functions.py @@ -0,0 +1,147 @@ +import collections.abc +import itertools +import pathlib +import re + +import jinja2 + + +def uniq_list(a: list) -> list: + return list(dict.fromkeys(a)) + + +def list_remove_non_str(a: list) -> list: + return list(itertools.filterfalse( + lambda x: not isinstance(x, str), a + )) + + +def list_remove_empty_str(a: list) -> list: + return list(itertools.filterfalse( + lambda x: len(x) == 0, a + )) + + +def uniq_str_list(a: list) -> list: + return uniq_list(list_remove_empty_str(a)) + + +def str_split_to_list(s: str, sep=r'\s+') -> list: + return list_remove_empty_str( + re.split(sep, s) + ) + + +def is_sequence(x) -> bool: + return isinstance(x, collections.abc.Sequence) + + +def is_mapping(x) -> bool: + return isinstance(x, collections.abc.Mapping) + + +def any_to_str_list(k) -> list: + if isinstance(k, str): + return [k] + + if is_sequence(k): + return [str(e) for e in k] + + return [str(k)] + + +def is_str_list_re_match(a: list, pattern, flags=0) -> bool: + return any(re.match(pattern, x, flags) for x in a) + + +def is_str_list_re_fullmatch(a: list, pattern, flags=0) -> bool: + return any(re.fullmatch(pattern, x, flags) for x in a) + + +def str_list_re_match(a: list, pattern, flags=0) -> list: + return [x for x in a if re.match(pattern, x, flags)] + + +def str_list_re_fullmatch(a: list, pattern, flags=0) -> list: + return [x for x in a if re.fullmatch(pattern, x, flags)] + + +def str_list_re_sub(a: list, pattern, repl, count=0, flags=0) -> list: + return [re.sub(pattern, repl, x, count, flags) for x in a] + + +@jinja2.pass_environment +def sh_like_file_to_list(j2env, file_in: str) -> list: + tpl = j2env.get_template(file_in) + text = pathlib.Path(tpl.filename).read_text(encoding='utf-8') + lines = re.split(r'\r\n', text) + return list(itertools.filterfalse( + lambda x: re.match(r'\s*#', x), lines + )) + + +def as_cgi_header(s: str) -> str: + return 'HTTP_' + re.sub('[^A-Z0-9]+', '_', s.upper()).strip('_') + + +def env_any_to_str_list(x) -> list: + if x is None: + return [] + h = {} + + def feed(k, v=None): + k = str(k) + if v is None: + k2, m, v2 = k.partition('=') + if m == '=': + k = k2 + v = v2 + if len(k) == 0: + return + if not re.fullmatch(r'[a-zA-Z_][a-zA-Z0-9_]*', k): + return + if k in h: + return + if v is None: + h[k] = v + else: + h[k] = str(v) + + if isinstance(x, str): + feed(x) + elif is_sequence(x): + for e in x: + feed(e) + elif is_mapping(x): + for k, v in x.items(): + feed(k, v) + else: + return [] + + r = [] + for k in sorted(h.keys()): + if h[k] is None: + r.append(k) + else: + r.append(f'{k}={h[k]}') + return r + + +J2CFG_FILTERS = [ + any_to_str_list, + as_cgi_header, + env_any_to_str_list, + is_mapping, + is_sequence, + is_str_list_re_fullmatch, + is_str_list_re_match, + list_remove_empty_str, + list_remove_non_str, + sh_like_file_to_list, + str_list_re_fullmatch, + str_list_re_match, + str_list_re_sub, + str_split_to_list, + uniq_list, + uniq_str_list, +] diff --git a/j2cfg/j2cfg/settings.py b/j2cfg/j2cfg/settings.py new file mode 100644 index 0000000..dc1e3b2 --- /dev/null +++ b/j2cfg/j2cfg/settings.py @@ -0,0 +1,26 @@ +J2CFG_TEMPLATE_EXT = '.j2' + +J2CFG_PATH = [ + '/etc/angie/j2cfg.dist', + '/etc/angie/j2cfg', + '/angie/j2cfg', +] + +J2CFG_PYTHON_MODULES = [ + 'itertools', + 'json', + 'os', + 'os.path', + 'pathlib', + 're', + 'sys', + # installed through pip + 'psutil', + 'netaddr', + 'wcmatch', +] + +J2CFG_JINJA_EXTENSIONS = [ + 'jinja2.ext.do', + 'jinja2.ext.loopcontrols', +] diff --git a/jinja2/j2common.py b/jinja2/j2common.py deleted file mode 100755 index ecf1420..0000000 --- a/jinja2/j2common.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -import importlib -import json -import os -import os.path -import sys -import yaml -import jinja2 - -ME = sys.argv[0] - -J2_MODULES_DEFAULT = 'os os.path sys netaddr pathlib psutil re wcmatch' -J2_SUFFIX = '.j2' -J2_CFG_PATHS = [ - '/angie/j2cfg', - '/etc/angie/j2cfg', -] -J2_CFG_EXTS = [ - 'yml', - 'yaml', - 'json', -] -J2_SEARCH_PATH = [ - '/etc/angie', - '/run/angie', - '/', -] -J2_EXT_LIST = [ - 'jinja2.ext.do', - 'jinja2.ext.loopcontrols', -] - -J2_MODULES = sorted(set( - os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ') -)) - -J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '') - -J2_KWARGS = {} - - -def merge_dict_from_file(filename): - if (not filename) or (str(filename) == ''): - return False - if not os.path.exists(filename): - return False - if not os.path.isfile(filename): - print( - f'{ME}: not a file, skipping: {filename}', - file=sys.stderr) - return False - if filename.endswith('.yml') or filename.endswith('.yaml'): - with open(filename, mode='r', encoding='utf-8') as fx: - x = yaml.safe_load(fx) - J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x - return True - if filename.endswith('.json'): - with open(filename, mode='r', encoding='utf-8') as fx: - x = json.load(fx) - J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x - return True - print( - f'{ME}: non-recognized name extension: {filename}', - file=sys.stderr) - return False - - -def merge_dict_default(): - for base in J2_CFG_PATHS: - for full in [base + '.' + ext for ext in J2_CFG_EXTS]: - if merge_dict_from_file(full): - break - continue - - -def render_error(msg, fail=True) -> bool: - if fail: - raise ValueError(msg) - print(f'{ME}: {msg}', file=sys.stderr) - return False - - -def render_file(file_in, file_out=None, fail=True): - if (not file_in) or (str(file_in) == ''): - return render_error( - 'argument "file_in" is empty', - fail) - if not os.path.exists(file_in): - return render_error( - f'file is missing: {file_in}', - fail) - if not os.path.isfile(file_in): - return render_error( - f'not a file: {file_in}', - fail) - - f_out = file_out - if not f_out: - if not file_in.endswith(J2_SUFFIX): - return render_error( - f'input file name extension mismatch: {file_in}', - fail) - f_out = os.path.splitext(file_in)[0] - - dirs = J2_SEARCH_PATH.copy() - for d in [os.path.dirname(file_in), os.getcwd()]: - if d not in dirs: - dirs.insert(0, d) - - j2_loader = jinja2.ChoiceLoader([ - jinja2.FileSystemLoader( - d, - encoding='utf-8', - followlinks=True, - ) for d in dirs - ]) - - j2_environ = jinja2.Environment( - loader=j2_loader, - extensions=J2_EXT_LIST, - ) - j2_template = j2_environ.get_template(file_in) - j2_stream = j2_template.stream(**J2_KWARGS) - j2_stream.disable_buffering() - - if os.path.lexists(f_out): - if os.path.islink(f_out) or (not os.path.isfile(f_out)): - return render_error( - f'output file is not safely writable: {f_out}', - fail) - if os.path.exists(f_out): - if os.path.samefile(file_in, f_out): - return render_error( - f'unable to process template inplace: {file_in}', - fail) - j2_stream.dump(f_out, encoding='utf-8') - return True - - -def init(): - kwa = {} - for m in J2_MODULES: - kwa[m] = importlib.import_module(m) - kwa['env'] = os.environ - kwa['cfg'] = {} - global J2_KWARGS - J2_KWARGS = kwa - - if J2_CONFIG != '': - if os.path.isfile(J2_CONFIG): - merge_dict_from_file(J2_CONFIG) - else: - print( - f'{ME}: J2_CONFIG does not exist, skipping: {J2_CONFIG}', - file=sys.stderr - ) - merge_dict_default() - else: - merge_dict_default() diff --git a/scripts/j2-multi b/scripts/j2cfg-multi similarity index 76% rename from scripts/j2-multi rename to scripts/j2cfg-multi index 4bfcbb8..9102305 100755 --- a/scripts/j2-multi +++ b/scripts/j2cfg-multi @@ -5,4 +5,4 @@ echo "# ${pfx}${0##*/}:" >&2 printf ' - %s\n' "$@" >&2 } -exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@" \ No newline at end of file +exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@" \ No newline at end of file diff --git a/scripts/j2-single b/scripts/j2cfg-single similarity index 75% rename from scripts/j2-single rename to scripts/j2cfg-single index 076b5a7..f2a8d5a 100755 --- a/scripts/j2-single +++ b/scripts/j2cfg-single @@ -4,4 +4,4 @@ [ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): " echo "# ${pfx}${0##*/}:${*:+ $*}" >&2 } -exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@" \ No newline at end of file +exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@" \ No newline at end of file