krd/angie-conv-image
1
0
Fork 0
Code

heavily rework template unrolling

Browse Source
This commit is contained in:
Konstantin Demin 2024-07-20 16:35:39 +03:00
parent 7298498885
commit 5d3307fe57
Signed by: krd
GPG Key ID: 4D56F87A8BA65FD0
45 changed files with 619 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.dockerignore
View File

@ -1 +1,2 @@
jinja2/__pycache__
j2cfg/__pycache__
j2cfg/j2cfg/__pycache__

10
Dockerfile
View File

@ -26,13 +26,13 @@ SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
COPY /jinja2/ /usr/local/lib/jinja2/
COPY /j2cfg/ /usr/local/lib/j2cfg/
ENV PYTHONDONTWRITEBYTECODE=''
## Python cache preseed
RUN python3 -m compileall -q -j 2 /usr/local/lib/jinja2/
RUN python3 -m compileall -q -j 2 /usr/local/lib/j2cfg/
RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
@ -51,7 +51,7 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
## Python cache warmup
RUN echo > /tmp/f.j2 ; \
j2-single /tmp/f.j2 ; \
j2cfg-single /tmp/f.j2 ; \
rm -f /tmp/f /tmp/f.j2
## Python cache adjustments
@ -76,7 +76,7 @@ COPY --from=certs /usr/local/share/ca-certificates/ /usr/local/share/ca-certif
COPY --from=pycache /usr/local/lib/ /usr/local/lib/
## already copied by statement above
# COPY /jinja2/ /usr/local/lib/jinja2/
# COPY /j2cfg/ /usr/local/lib/j2cfg/
ENV ANGIE_MODULES_DIR=/usr/lib/angie/modules
@ -141,7 +141,7 @@ RUN install -d -o angie -g angie -m 03777 /angie /run/angie ; \
ln -sv /run/angie/lock lock.d ; \
ln -sv ${ANGIE_MODULES_DIR} modules.dist ; \
## hyper-modular paths:
data='conf mod modules njs site snip static' ; \
data='conf j2cfg mod modules njs site snip static' ; \
vardata='cache lib log' ; \
for n in ${data} ; do \
for d in "$n" "$n.dist" ; do \

33
angie/conf.dist/core-preserve-env.conf.j2
View File

@ -1,33 +1,18 @@
{#- prologue -#}
{%- set penv = [] -%}
{%- if cfg.preserve_env -%}
{%- set penv = cfg.preserve_env -%}
{%- if penv is string -%}
{%- set penv = [penv] -%}
{%- elif penv is iterable -%}
{#- {%- set penv = penv -%} -#}
{%- else -%}
{%- set penv = [penv|string()] -%}
{%- endif -%}
{%- endif -%}
{%- set have = namespace() -%}
{%- set have.tz = false -%}
{%- set have.malloc_arena = false -%}
{#- scan -#}
{%- for v in penv -%}
{%- set have.tz = have.tz or re.match('TZ(=|$)', v|string()) -%}
{%- set have.malloc_arena = have.malloc_arena or re.match('MALLOC_ARENA_MAX(=|$)', v|string()) -%}
{%- endfor -%}
{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%}
{%- set have_tz = preserve_env|is_str_list_re_match('TZ(=|$)') -%}
{%- set have_malloc_arena = preserve_env|is_str_list_re_match('MALLOC_ARENA_MAX(=|$)') -%}
{#- main part -#}
{%- if not have.tz -%}
{%- if not have_tz -%}
env TZ;
{% endif %}
{%- if not have.malloc_arena -%}
{%- if not have_malloc_arena -%}
env MALLOC_ARENA_MAX;
{% endif %}
{%- for v in penv -%}
{%- if re.search("(\"|'|\\s)", v|string()) %}
env {{ (v|string()).__repr__() }};
{%- for v in preserve_env -%}
{%- if re.search("(\"|'|\\s)", v) %}
{#- TODO: investigate corrent escape behavior for Angie/nginx -#}
env {{ v.__repr__() }};
{%- else %}
env {{ v }};
{%- endif %}

15
angie/j2cfg.dist/compress-types.yml Normal file
View File

@ -0,0 +1,15 @@
compress_types:
- application/atom+xml
- application/javascript
- application/json
- application/vnd.api+json
- application/rss+xml
- application/x-javascript
- application/xhtml+xml
- application/xml
- image/svg+xml
- image/x-icon
- text/css
- text/javascript
- text/plain
- text/xml

7
angie/j2cfg.dist/core-preserve-environment.txt.j2 Normal file
View File

@ -0,0 +1,7 @@
{#- prologue -#}
{%- set preserve_env = ( j2cfg.core_preserve_environment or [] )|env_any_to_str_list -%}
{%- set preserve_vars = preserve_env|str_list_re_fullmatch('[^=]+') -%}
{#- main part -#}
{% for v in preserve_vars -%}
{{ v }}
{% endfor -%}

5
angie/snip.dist/brotli/buffers.conf Normal file
View File

@ -0,0 +1,5 @@
brotli_comp_level 5; # default: 6
brotli_window 64k; # default: 512k
brotli_min_length 1024;
brotli_buffers 32 16k;

1
angie/snip.dist/brotli/gzip.conf Normal file
View File

@ -0,0 +1 @@
include snip.d/gzip/vary.conf;

8
angie/snip.dist/brotli/types.conf.j2 Normal file
View File

@ -0,0 +1,8 @@
{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%}
{%- if mime_types %}
brotli_types
{%- for t in mime_types %}
{{ t }}
{%- endfor %}
;
{%- endif %}

4
angie/snip.dist/gzip/buffers.conf Normal file
View File

@ -0,0 +1,4 @@
gzip_comp_level 2; # default: 1
gzip_min_length 1024;
gzip_buffers 32 16k;

1
angie/snip.dist/gzip/proxied.conf Normal file
View File

@ -0,0 +1 @@
gzip_proxied any;

8
angie/snip.dist/gzip/types.conf.j2 Normal file
View File

@ -0,0 +1,8 @@
{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%}
{%- if mime_types %}
gzip_types
{%- for t in mime_types %}
{{ t }}
{%- endfor %}
;
{%- endif %}

1
angie/snip.dist/gzip/vary.conf Normal file
View File

@ -0,0 +1 @@
gzip_vary on;

2
angie/snip.dist/http-brotli.conf
View File

@ -1,2 +1,2 @@
include snip.d/http-brotli.modconf;
include snip.d/brotli/*.conf;
brotli on;

14
angie/snip.dist/http-brotli.modconf.j2
View File

@ -1,14 +0,0 @@
{%- from 'mime-types.compress.j2inc' import mime_types with context -%}
## default is 6
brotli_comp_level 5;
## default is 512k
brotli_window 64k;
brotli_min_length 1024;
brotli_buffers 32 16k;
brotli_types
## sourced from mime-types.compress.txt
{{ mime_types | indent(4) }}
;

0
angie/snip.dist/http-gunzip → angie/snip.dist/http-gunzip.modconf
View File

2
angie/snip.dist/http-gzip.conf
View File

@ -1,2 +1,2 @@
include snip.d/http-gzip.modconf;
include snip.d/gzip/*.conf;
gzip on;

15
angie/snip.dist/http-gzip.modconf.j2
View File

@ -1,15 +0,0 @@
{%- from 'mime-types.compress.j2inc' import mime_types with context -%}
## default is 1
gzip_comp_level 2;
gzip_min_length 1024;
gzip_buffers 32 16k;
gzip_vary on;
gzip_proxied any;
gzip_types
## sourced from mime-types.compress.txt
{{ mime_types | indent(4) }}
;

2
angie/snip.dist/http-zstd.conf
View File

@ -1,2 +1,2 @@
include snip.d/http-zstd.modconf;
include snip.d/zstd/*.conf;
zstd on;

12
angie/snip.dist/http-zstd.modconf.j2
View File

@ -1,12 +0,0 @@
{%- from 'mime-types.compress.j2inc' import mime_types with context -%}
## default is 1
zstd_comp_level 2;
zstd_min_length 1024;
zstd_buffers 32 16k;
zstd_types
## sourced from mime-types.compress.txt
{{ mime_types | indent(4) }}
;

2
angie/snip.dist/mime-types.compress.j2inc
View File

@ -1,2 +0,0 @@
{%- set mime_file = pathlib.Path(os.path.join(os.getenv('NGX_MERGED_ROOT'), 'snip/mime-types.compress.txt')) -%}
{%- set mime_types = mime_file.read_text() -%}

14
angie/snip.dist/mime-types.compress.txt
View File

@ -1,14 +0,0 @@
application/atom+xml
application/javascript
application/json
application/vnd.api+json
application/rss+xml
application/x-javascript
application/xhtml+xml
application/xml
image/svg+xml
image/x-icon
text/css
text/javascript
text/plain
text/xml

4
angie/snip.dist/zstd/buffers.conf Normal file
View File

@ -0,0 +1,4 @@
zstd_comp_level 2; # default: 1
zstd_min_length 1024;
zstd_buffers 32 16k;

1
angie/snip.dist/zstd/gzip.conf Normal file
View File

@ -0,0 +1 @@
include snip.d/gzip/vary.conf;

8
angie/snip.dist/zstd/types.conf.j2 Normal file
View File

@ -0,0 +1,8 @@
{%- set mime_types = ( j2cfg.compress_types or [] )|any_to_str_list|uniq_str_list -%}
{%- if mime_types %}
zstd_types
{%- for t in mime_types %}
{{ t }}
{%- endfor %}
;
{%- endif %}

14
image-entry.d/00-common.envsh
View File

@ -82,7 +82,7 @@ untemplate_path() {
"${volume_root}"/* | /etc/angie/run/* )
strip_suffix "$1" "$2"
;;
/etc/angie/conf.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* )
/etc/angie/conf.d/* | /etc/angie/j2cfg.d/* | /etc/angie/mod.d/* | /etc/angie/modules.d/* | /etc/angie/njs.d/* | /etc/angie/site.d/* | /etc/angie/snip.d/* )
strip_suffix "$1" "$2"
;;
/etc/angie/static.d/* )
@ -151,8 +151,8 @@ expand_file_envsubst() {
return ${__r}
}
expand_file_jinja() {
j2-single "$@" || return $?
expand_file_j2cfg() {
j2cfg-single "$@" || return $?
}
expand_dir_envsubst() {
@ -187,7 +187,7 @@ expand_dir_envsubst() {
return ${__ret}
}
expand_dir_jinja() {
expand_dir_j2cfg() {
__template_list=$(mktemp) || return
find "$@" -follow -type f -name '*.j2' -printf '%p\0' \
@ -196,7 +196,7 @@ expand_dir_jinja() {
__ret=0
if [ -s "${__template_list}" ] ; then
xargs -0r -n 1000 -a "${__template_list}" \
j2-multi < /dev/null || __ret=1
j2cfg-multi < /dev/null || __ret=1
fi
rm -f "${__template_list}" ; unset __template_list
@ -212,6 +212,10 @@ remap_path() {
/etc/angie/conf.dist/* ) echo "${2:-/etc/angie/conf.d}${1#/etc/angie/conf.dist}" ;;
/etc/angie/conf/* ) echo "${2:-/etc/angie/conf.d}${1#/etc/angie/conf}" ;;
/angie/conf/* ) echo "${2:-/etc/angie/conf.d}${1#/angie/conf}" ;;
## j2cfg
/etc/angie/j2cfg.dist/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/etc/angie/j2cfg.dist}" ;;
/etc/angie/j2cfg/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/etc/angie/j2cfg}" ;;
/angie/j2cfg/* ) echo "${2:-/etc/angie/j2cfg.d}${1#/angie/j2cfg}" ;;
## mod
/etc/angie/mod.dist/* ) echo "${2:-/etc/angie/mod.d}${1#/etc/angie/mod.dist}" ;;
/etc/angie/mod/* ) echo "${2:-/etc/angie/mod.d}${1#/etc/angie/mod}" ;;

28
image-entry.d/01-defaults.envsh
View File

@ -3,19 +3,12 @@
## NB: NGX_DEBUG is set via image build script
set -a
NGX_STRICT_LOAD=$(gobool_to_int "${NGX_STRICT_LOAD:-1}" 1)
NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0)
NGX_HTTP=$(gobool_to_int "${NGX_HTTP:-1}" 1)
NGX_MAIL=$(gobool_to_int "${NGX_MAIL:-0}" 0)
NGX_STREAM=$(gobool_to_int "${NGX_STREAM:-0}" 0)
NGX_STRICT_LOAD=$(gobool_to_int "${NGX_STRICT_LOAD:-1}" 1)
NGX_CORE_MODULES="${NGX_CORE_MODULES:-}"
NGX_CORE_EVENTS_SNIPPETS="${NGX_CORE_EVENTS_SNIPPETS:-}"
NGX_CORE_SNIPPETS="${NGX_CORE_SNIPPETS:-}"
NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0)
set +a
if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then
@ -24,3 +17,20 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then
log_always 'Angie is almost completely TURNED OFF'
log_always '======================================'
fi
unset default_dirs_merge default_dirs_link
default_dirs_merge='conf j2cfg mod modules njs site snip'
default_dirs_link=''
if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then
NGX_DIRS_MERGE="${NGX_DIRS_MERGE:-} static"
else
NGX_DIRS_LINK="${NGX_DIRS_LINK:-} static"
fi
set -a
NGX_DIRS_MERGE=$(sort_dedup_list "${default_dirs_merge} ${NGX_DIRS_MERGE:-}")
NGX_DIRS_LINK=$(sort_dedup_list "${default_dirs_link} ${NGX_DIRS_LINK:-}")
set +a
unset default_dirs_merge default_dirs_link

9
image-entry.d/10-core.envsh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
set -a
NGX_CORE_MODULES="${NGX_CORE_MODULES:-}"
NGX_CORE_EVENTS_SNIPPETS="${NGX_CORE_EVENTS_SNIPPETS:-}"
NGX_CORE_SNIPPETS="${NGX_CORE_SNIPPETS:-}"
set +a

8
image-entry.d/10-core-modules.envsh → image-entry.d/11-core-modules.envsh
View File

@ -11,7 +11,7 @@ for i in ${NGX_CORE_MODULES:-} ; do
if is_builtin_module core "$i" ; then
log "$i is builtin module, moving to snippets"
core_snippets="${core_snippets}${core_snippets:+ }$i"
core_snippets="${core_snippets} $i"
continue
fi
@ -27,12 +27,10 @@ for i in ${NGX_CORE_MODULES:-} ; do
done
unset i
## sort and remove duplicates
core_snippets=$(sort_dedup_list "${core_snippets}")
set -a
NGX_CORE_MODULES="${core_modules}"
NGX_CORE_SNIPPETS="${core_snippets}"
NGX_CORE_SNIPPETS=$(sort_dedup_list "${core_snippets}")
NGX_CORE_EVENTS_SNIPPETS=$(sort_dedup_list "${NGX_CORE_EVENTS_SNIPPETS}")
set +a
unset core_modules core_snippets

0
image-entry.d/11-core-user.envsh → image-entry.d/12-core-user.envsh
View File

0
image-entry.d/12-core-worker-defaults.envsh → image-entry.d/13-core-worker-defaults.envsh
View File

6
image-entry.d/21-http-modules.envsh
View File

@ -21,7 +21,7 @@ if [ "${NGX_HTTP}" = 1 ] ; then
if is_builtin_module http "$i" ; then
log "$i is builtin module, moving to snippets"
http_snippets="${http_snippets}${http_snippets:+ }$i"
http_snippets="${http_snippets} $i"
continue
fi
@ -37,11 +37,9 @@ if [ "${NGX_HTTP}" = 1 ] ; then
done
unset i
http_snippets=$(sort_dedup_list "${http_snippets}")
set -a
NGX_HTTP_MODULES="${http_modules}"
NGX_HTTP_SNIPPETS="${http_snippets}"
NGX_HTTP_SNIPPETS=$(sort_dedup_list "${http_snippets}")
set +a
unset http_modules http_snippets

6
image-entry.d/31-mail-modules.envsh
View File

@ -12,7 +12,7 @@ if [ "${NGX_MAIL}" = 1 ] ; then
if is_builtin_module mail "$i" ; then
log "$i is builtin module, moving to snippets"
mail_snippets="${mail_snippets}${mail_snippets:+ }$i"
mail_snippets="${mail_snippets} $i"
continue
fi
@ -28,11 +28,9 @@ if [ "${NGX_MAIL}" = 1 ] ; then
done
unset i
mail_snippets=$(sort_dedup_list "${mail_snippets}")
set -a
NGX_MAIL_MODULES="${mail_modules}"
NGX_MAIL_SNIPPETS="${mail_snippets}"
NGX_MAIL_SNIPPETS=$(sort_dedup_list "${mail_snippets}")
set +a
unset mail_modules mail_snippets

6
image-entry.d/41-stream-modules.envsh
View File

@ -12,7 +12,7 @@ if [ "${NGX_STREAM}" = 1 ] ; then
if is_builtin_module stream "$i" ; then
log "$i is builtin module, moving to snippets"
stream_snippets="${stream_snippets}${stream_snippets:+ }$i"
stream_snippets="${stream_snippets} $i"
continue
fi
@ -28,11 +28,9 @@ if [ "${NGX_STREAM}" = 1 ] ; then
done
unset i
stream_snippets=$(sort_dedup_list "${stream_snippets}")
set -a
NGX_STREAM_MODULES="${stream_modules}"
NGX_STREAM_SNIPPETS="${stream_snippets}"
NGX_STREAM_SNIPPETS=$(sort_dedup_list "${stream_snippets}")
set +a
unset stream_modules stream_snippets

5
image-entry.d/52-merge-tree.sh
View File

@ -5,10 +5,9 @@ set -ef
[ -d "${merged_root}" ] || install -d "${merged_root}"
dirs='conf mod modules njs site snip'
[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
for n in ${NGX_DIRS_MERGE} ; do
[ -n "$n" ] || continue
for n in ${dirs} ; do
merged_dir="${merged_root}/$n"
while read -r old_path ; do
[ -n "${old_path}" ] || continue

32
image-entry.d/53-expand-templates.sh
View File

@ -5,31 +5,41 @@ set -f
[ "${NGX_STRICT_LOAD}" = 0 ] || set -e
export NGX_MERGED_ROOT="${merged_root}"
cd "${merged_root}/"
expand_error_delim() {
IEP_TRACE=0 log_always ' ----------------------------------- '
}
expand_error() {
[ "${expand_error_seen:-}" = 1 ] || log_always 'template expansion has failed'
[ "${expand_error_seen:-}" != 1 ] || return
expand_error_seen=1
expand_error_delim
log_always 'template expansion has failed'
if [ "${NGX_STRICT_LOAD}" = 1 ] ; then
t=10
t=15
log_always "injecting delay for $t seconds"
expand_error_delim
sleep $t
exit 1
fi
expand_error_delim
}
dirs='conf mod modules njs site snip'
[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
merge_dirs=
for n in ${dirs} ; do
merged_dir="${merged_root}/$n"
[ -d "${merged_dir}" ] || continue
for n in ${NGX_DIRS_MERGE} ; do
[ -n "$n" ] || continue
[ -d "$n" ] || continue
merge_dirs="${merge_dirs} ${merged_dir}/"
merge_dirs="${merge_dirs} $n/"
done
expand_dir_envsubst ${merge_dirs} || expand_error
expand_dir_jinja ${merge_dirs} || expand_error
set -a
J2CFG_PATH="${merged_root}/j2cfg"
J2CFG_SEARCH_PATH="${merged_root}"
set -a
expand_dir_j2cfg ${merge_dirs} || expand_error
exit 0

28
image-entry.d/54-combine-tree.sh
View File

@ -30,7 +30,7 @@ load_error() {
load_error_delim
log_always 'tree combine has failed'
if [ "${NGX_STRICT_LOAD}" = 1 ] ; then
t=10
t=15
log_always "injecting delay for $t seconds"
load_error_delim
sleep $t
@ -53,13 +53,11 @@ done
## provide same symlinks as upstream (both Angie and nginx) docker images do
d="${target_root}/log"
[ -e "$d/access.log" ] || ln_s /dev/stdout "$d/access.log"
[ -e "$d/error.log" ] || ln_s /dev/stderr "$d/error.log"
[ -e "$d/access.log" ] || ln_s /dev/stdout "$d/access.log" || load_error
[ -e "$d/error.log" ] || ln_s /dev/stderr "$d/error.log" || load_error
## NB: if any error occurs above then configuration is merely empty and/or broken
dirs='conf mod modules njs site snip'
[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
while read -r old_path ; do
[ -n "${old_path}" ] || continue
@ -73,7 +71,9 @@ while read -r old_path ; do
done <<-EOF
$(
set +e
for n in ${dirs} ; do
for n in ${NGX_DIRS_MERGE} ; do
[ -n "$n" ] || continue
[ -d "${merged_root}/$n" ] || continue
find "${merged_root}/$n/" ! -type d
done \
@ -81,17 +81,23 @@ $(
-e "^${merged_root}/(mod|snip)/.+\.load\$" \
-e "^${merged_root}/mod/[^/]+\.preseed\$" \
| sort -V
set -e
)
EOF
if [ "${NGX_PROCESS_STATIC}" = 0 ] ; then
for d in /angie/static /etc/angie/static /etc/angie/static.dist ; do
for n in ${NGX_DIRS_LINK} ; do
[ -n "$n" ] || continue
if [ -e "${target_root}/$n" ] ; then continue ; fi
for d in "/angie/$n" "/etc/angie/$n" "/etc/angie/$n.dist" ; do
[ -d "$d" ] || continue
ln_s "$d" "${target_root}/static"
ln_s "$d" "${target_root}/$n"
break
done
fi
[ -d "${target_root}/$n" ] || {
log "missing required directory: ${target_root}/$n"
}
done
## Angie modules are loaded in [strict] order!
combine_modules() {

41
image-entry.d/99-cleanup-env.envsh Executable file
View File

@ -0,0 +1,41 @@
#!/bin/sh
__set="$-"
set +e
if [ "${IEP_TRACE}" = 1 ] ; then
log_always "NOT going to unset following variables:"
sed -E '/^./s,^,- ,' >&2
else
unset __env
while read -r __env ; do
[ -n "${__env}" ] || continue
case "${__env}" in
*\'* )
log "skipping variable (malformed): ${__env}" >&2
continue
;;
esac
log "unsetting variable: ${__env}"
unset "${__env}"
done
unset __env
fi <<-EOF
$(
cat /proc/self/environ \
| sed -zEn '/^([^=]+).*$/s//\1/p' \
| xargs -0r printf '%q\n' \
| {
f="${target_root}/j2cfg/core-preserve-environment.txt"
[ -s "$f" ] || exec cat
grep -Fxv -f "$f"
} \
| grep -E \
-e '^(NGX|PYTHON)' \
| sort -uV
)
EOF
[ -z "${__set}" ] || set -"${__set}"
unset __set

9
jinja2/j2-multi.py → j2cfg/j2cfg-multi.py
View File

@ -3,19 +3,18 @@
import os.path
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import j2common
def main():
if len(sys.argv) < 2:
raise ValueError('not enough arguments (min: 1)')
j2common.init()
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import j2cfg
r = j2cfg.J2cfg(strict=False)
ret = 0
for f in sys.argv[1:]:
if not j2common.render_file(f, None, False):
if not r.render_file(f, None):
ret = 1
sys.exit(ret)

14
jinja2/j2-single.py → j2cfg/j2cfg-single.py
View File

@ -3,9 +3,6 @@
import os.path
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import j2common
def main():
if len(sys.argv) < 2:
@ -16,13 +13,14 @@ def main():
if not sys.argv[1]:
raise ValueError('specify input file')
if len(sys.argv) == 3:
if not sys.argv[2]:
raise ValueError('specify output file')
if (len(sys.argv) == 3) and (not sys.argv[2]):
raise ValueError('specify output file')
j2common.init()
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import j2cfg
j2common.render_file(*sys.argv[1:])
r = j2cfg.J2cfg()
r.render_file(*sys.argv[1:])
sys.exit(0)

225
j2cfg/j2cfg/__init__.py Normal file
View File

@ -0,0 +1,225 @@
import importlib
import json
import os
import os.path
import sys
import jinja2
import wcmatch.wcmatch
import yaml
from .functions import *
from .settings import *
J2CFG_CONFIG_EXT = ['yml', 'yaml', 'json']
class J2cfg:
def __init__(self, strict=True, config=None, config_path=None,
modules=None, search_path=None, template_suffix=None):
self.strict = strict
if not isinstance(self.strict, bool):
self.strict = True
self.config_file = config or os.getenv('J2CFG_CONFIG')
if self.config_file is not None:
self.config_file = str(self.config_file)
self.config_path = config_path
if self.config_path is None:
self.config_path = os.getenv('J2CFG_PATH')
if self.config_path is not None:
self.config_path = str_split_to_list(self.config_path, ':')
if self.config_path is None:
self.config_path = J2CFG_PATH.copy()
else:
self.config_path = any_to_str_list(self.config_path)
self.config_path = uniq_str_list(self.config_path)
self.search_path = search_path
if self.search_path is None:
self.search_path = os.getenv('J2CFG_SEARCH_PATH')
if self.search_path is not None:
self.search_path = str_split_to_list(self.search_path, ':')
if self.search_path is None:
self.search_path = self.config_path.copy()
else:
self.search_path = any_to_str_list(self.search_path)
self.search_path = uniq_str_list(self.search_path)
# RFC: should we use the current working directory early?
for d in [os.getcwd()]:
if d not in self.search_path:
self.search_path.insert(0, d)
self.modules = modules or os.getenv('J2CFG_MODULES')
if self.modules is None:
self.modules = J2CFG_PYTHON_MODULES.copy()
else:
if isinstance(self.modules, str):
self.modules = str_split_to_list(self.modules)
else:
self.modules = any_to_str_list(self.modules)
self.modules = uniq_str_list(self.modules)
self.template_suffix = template_suffix or os.getenv('J2CFG_SUFFIX')
if self.template_suffix is None:
self.template_suffix = J2CFG_TEMPLATE_EXT
else:
self.template_suffix = str(self.template_suffix)
if self.template_suffix == '':
self.template_suffix = J2CFG_TEMPLATE_EXT
if not self.template_suffix.startswith('.'):
self.template_suffix = '.' + self.template_suffix
self.kwargs = {
'env': os.environ,
'j2cfg': {}
}
for m in self.modules:
if m in self.kwargs:
print(f'J2cfg: kwargs already has {m} key',
file=sys.stderr)
continue
self.kwargs[m] = importlib.import_module(m)
def merge_dict_from_file(filename):
if filename is None:
return False
f = str(filename)
if f == '':
return False
if not os.path.exists(f):
return False
if not os.path.isfile(f):
print(
f'J2cfg: not a file, skipping: {filename}',
file=sys.stderr)
return False
if f.endswith('.yml') or f.endswith('.yaml'):
with open(f, mode='r', encoding='utf-8') as fx:
x = yaml.safe_load(fx)
self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x
return True
if f.endswith('.json'):
with open(f, mode='r', encoding='utf-8') as fx:
x = json.load(fx)
self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x
return True
print(
f'J2cfg: non-recognized name extension: {filename}',
file=sys.stderr)
return False
def merge_dict_default():
search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT])
search_flags = wcmatch.wcmatch.RECURSIVE | wcmatch.wcmatch.SYMLINKS
for d in self.config_path:
if not os.path.isdir(d):
continue
for f in wcmatch.wcmatch.WcMatch(d, search_pattern,
flags=search_flags).imatch():
merge_dict_from_file(f)
if self.config_file is None:
merge_dict_default()
else:
if os.path.isfile(self.config_file):
merge_dict_from_file(self.config_file)
else:
print(
'J2cfg: J2cfg config file does not exist, skipping: '
+ f'{self.config_file}',
file=sys.stderr
)
self.j2fs_loaders = {
d: jinja2.FileSystemLoader(
d, encoding='utf-8', followlinks=True,
) for d in self.search_path
}
self.j2env = jinja2.Environment(
extensions=J2CFG_JINJA_EXTENSIONS,
loader=jinja2.ChoiceLoader([
self.j2fs_loaders[d] for d in self.search_path
]),
)
def init_env(e: jinja2.Environment):
for s in J2CFG_FILTERS:
n = s.__name__
if n in e.filters:
print(f'J2cfg: filters already has {n} key',
file=sys.stderr)
continue
e.filters[n] = s
init_env(self.j2env)
def ensure_fs_loader_for(self, directory: str):
if directory in self.j2fs_loaders:
return
self.j2fs_loaders[directory] = jinja2.FileSystemLoader(
directory, encoding='utf-8', followlinks=True,
)
def render_file(self, file_in, file_out=None) -> bool:
def render_error(msg) -> bool:
if self.strict:
raise ValueError(msg)
print(f'J2cfg: {msg}', file=sys.stderr)
return False
if file_in is None:
return render_error(
'argument "file_in" is None')
f_in = str(file_in)
if f_in == '':
return render_error(
'argument "file_in" is empty')
if not os.path.exists(f_in):
return render_error(
f'file is missing: {file_in}')
if not os.path.isfile(f_in):
return render_error(
f'not a file: {file_in}')
f_out = file_out
if f_out is None:
if not f_in.endswith(self.template_suffix):
return render_error(
f'input file name extension mismatch: {file_in}')
f_out = os.path.splitext(f_in)[0]
dirs = self.search_path.copy()
for d in [os.getcwd(), os.path.dirname(f_in)]:
if d in dirs:
continue
self.ensure_fs_loader_for(d)
dirs.insert(0, d)
j2_environ = self.j2env.overlay(loader=jinja2.ChoiceLoader([
self.j2fs_loaders[d] for d in dirs
]))
j2_template = j2_environ.get_template(f_in)
rendered = j2_template.render(**self.kwargs)
if os.path.lexists(f_out):
if os.path.islink(f_out) or (not os.path.isfile(f_out)):
return render_error(
f'output file is not safely writable: {f_out}')
if os.path.exists(f_out):
if os.path.samefile(f_in, f_out):
return render_error(
f'unable to process template inplace: {file_in}')
with open(f_out, mode='w', encoding='utf-8') as f:
f.write(rendered)
return True

147
j2cfg/j2cfg/functions.py Normal file
View File

@ -0,0 +1,147 @@
import collections.abc
import itertools
import pathlib
import re
import jinja2
def uniq_list(a: list) -> list:
return list(dict.fromkeys(a))
def list_remove_non_str(a: list) -> list:
return list(itertools.filterfalse(
lambda x: not isinstance(x, str), a
))
def list_remove_empty_str(a: list) -> list:
return list(itertools.filterfalse(
lambda x: len(x) == 0, a
))
def uniq_str_list(a: list) -> list:
return uniq_list(list_remove_empty_str(a))
def str_split_to_list(s: str, sep=r'\s+') -> list:
return list_remove_empty_str(
re.split(sep, s)
)
def is_sequence(x) -> bool:
return isinstance(x, collections.abc.Sequence)
def is_mapping(x) -> bool:
return isinstance(x, collections.abc.Mapping)
def any_to_str_list(k) -> list:
if isinstance(k, str):
return [k]
if is_sequence(k):
return [str(e) for e in k]
return [str(k)]
def is_str_list_re_match(a: list, pattern, flags=0) -> bool:
return any(re.match(pattern, x, flags) for x in a)
def is_str_list_re_fullmatch(a: list, pattern, flags=0) -> bool:
return any(re.fullmatch(pattern, x, flags) for x in a)
def str_list_re_match(a: list, pattern, flags=0) -> list:
return [x for x in a if re.match(pattern, x, flags)]
def str_list_re_fullmatch(a: list, pattern, flags=0) -> list:
return [x for x in a if re.fullmatch(pattern, x, flags)]
def str_list_re_sub(a: list, pattern, repl, count=0, flags=0) -> list:
return [re.sub(pattern, repl, x, count, flags) for x in a]
@jinja2.pass_environment
def sh_like_file_to_list(j2env, file_in: str) -> list:
tpl = j2env.get_template(file_in)
text = pathlib.Path(tpl.filename).read_text(encoding='utf-8')
lines = re.split(r'\r\n', text)
return list(itertools.filterfalse(
lambda x: re.match(r'\s*#', x), lines
))
def as_cgi_header(s: str) -> str:
return 'HTTP_' + re.sub('[^A-Z0-9]+', '_', s.upper()).strip('_')
def env_any_to_str_list(x) -> list:
if x is None:
return []
h = {}
def feed(k, v=None):
k = str(k)
if v is None:
k2, m, v2 = k.partition('=')
if m == '=':
k = k2
v = v2
if len(k) == 0:
return
if not re.fullmatch(r'[a-zA-Z_][a-zA-Z0-9_]*', k):
return
if k in h:
return
if v is None:
h[k] = v
else:
h[k] = str(v)
if isinstance(x, str):
feed(x)
elif is_sequence(x):
for e in x:
feed(e)
elif is_mapping(x):
for k, v in x.items():
feed(k, v)
else:
return []
r = []
for k in sorted(h.keys()):
if h[k] is None:
r.append(k)
else:
r.append(f'{k}={h[k]}')
return r
J2CFG_FILTERS = [
any_to_str_list,
as_cgi_header,
env_any_to_str_list,
is_mapping,
is_sequence,
is_str_list_re_fullmatch,
is_str_list_re_match,
list_remove_empty_str,
list_remove_non_str,
sh_like_file_to_list,
str_list_re_fullmatch,
str_list_re_match,
str_list_re_sub,
str_split_to_list,
uniq_list,
uniq_str_list,
]

26
j2cfg/j2cfg/settings.py Normal file
View File

@ -0,0 +1,26 @@
J2CFG_TEMPLATE_EXT = '.j2'
J2CFG_PATH = [
'/etc/angie/j2cfg.dist',
'/etc/angie/j2cfg',
'/angie/j2cfg',
]
J2CFG_PYTHON_MODULES = [
'itertools',
'json',
'os',
'os.path',
'pathlib',
're',
'sys',
# installed through pip
'psutil',
'netaddr',
'wcmatch',
]
J2CFG_JINJA_EXTENSIONS = [
'jinja2.ext.do',
'jinja2.ext.loopcontrols',
]

159
jinja2/j2common.py
View File

@ -1,159 +0,0 @@
#!/usr/bin/env python3
import importlib
import json
import os
import os.path
import sys
import yaml
import jinja2
ME = sys.argv[0]
J2_MODULES_DEFAULT = 'os os.path sys netaddr pathlib psutil re wcmatch'
J2_SUFFIX = '.j2'
J2_CFG_PATHS = [
'/angie/j2cfg',
'/etc/angie/j2cfg',
]
J2_CFG_EXTS = [
'yml',
'yaml',
'json',
]
J2_SEARCH_PATH = [
'/etc/angie',
'/run/angie',
'/',
]
J2_EXT_LIST = [
'jinja2.ext.do',
'jinja2.ext.loopcontrols',
]
J2_MODULES = sorted(set(
os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ')
))
J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '')
J2_KWARGS = {}
def merge_dict_from_file(filename):
if (not filename) or (str(filename) == ''):
return False
if not os.path.exists(filename):
return False
if not os.path.isfile(filename):
print(
f'{ME}: not a file, skipping: {filename}',
file=sys.stderr)
return False
if filename.endswith('.yml') or filename.endswith('.yaml'):
with open(filename, mode='r', encoding='utf-8') as fx:
x = yaml.safe_load(fx)
J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
return True
if filename.endswith('.json'):
with open(filename, mode='r', encoding='utf-8') as fx:
x = json.load(fx)
J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
return True
print(
f'{ME}: non-recognized name extension: {filename}',
file=sys.stderr)
return False
def merge_dict_default():
for base in J2_CFG_PATHS:
for full in [base + '.' + ext for ext in J2_CFG_EXTS]:
if merge_dict_from_file(full):
break
continue
def render_error(msg, fail=True) -> bool:
if fail:
raise ValueError(msg)
print(f'{ME}: {msg}', file=sys.stderr)
return False
def render_file(file_in, file_out=None, fail=True):
if (not file_in) or (str(file_in) == ''):
return render_error(
'argument "file_in" is empty',
fail)
if not os.path.exists(file_in):
return render_error(
f'file is missing: {file_in}',
fail)
if not os.path.isfile(file_in):
return render_error(
f'not a file: {file_in}',
fail)
f_out = file_out
if not f_out:
if not file_in.endswith(J2_SUFFIX):
return render_error(
f'input file name extension mismatch: {file_in}',
fail)
f_out = os.path.splitext(file_in)[0]
dirs = J2_SEARCH_PATH.copy()
for d in [os.path.dirname(file_in), os.getcwd()]:
if d not in dirs:
dirs.insert(0, d)
j2_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader(
d,
encoding='utf-8',
followlinks=True,
) for d in dirs
])
j2_environ = jinja2.Environment(
loader=j2_loader,
extensions=J2_EXT_LIST,
)
j2_template = j2_environ.get_template(file_in)
j2_stream = j2_template.stream(**J2_KWARGS)
j2_stream.disable_buffering()
if os.path.lexists(f_out):
if os.path.islink(f_out) or (not os.path.isfile(f_out)):
return render_error(
f'output file is not safely writable: {f_out}',
fail)
if os.path.exists(f_out):
if os.path.samefile(file_in, f_out):
return render_error(
f'unable to process template inplace: {file_in}',
fail)
j2_stream.dump(f_out, encoding='utf-8')
return True
def init():
kwa = {}
for m in J2_MODULES:
kwa[m] = importlib.import_module(m)
kwa['env'] = os.environ
kwa['cfg'] = {}
global J2_KWARGS
J2_KWARGS = kwa
if J2_CONFIG != '':
if os.path.isfile(J2_CONFIG):
merge_dict_from_file(J2_CONFIG)
else:
print(
f'{ME}: J2_CONFIG does not exist, skipping: {J2_CONFIG}',
file=sys.stderr
)
merge_dict_default()
else:
merge_dict_default()

2
scripts/j2-multi → scripts/j2cfg-multi
View File

@ -5,4 +5,4 @@
echo "# ${pfx}${0##*/}:" >&2
printf ' - %s\n' "$@" >&2
}
exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"
exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@"

2
scripts/j2-single → scripts/j2cfg-single
View File

@ -4,4 +4,4 @@
[ "${IEP_TRACE}" = 0 ] || pfx="$(date +'%Y-%m-%d %H:%M:%S.%03N %z'): "
echo "# ${pfx}${0##*/}:${*:+ $*}" >&2
}
exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"
exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@"