krd/angie-conv-image
1
0
Fork 0
Code

rework template unrolling

Browse Source
This commit is contained in:
Konstantin Demin 2024-07-16 02:43:08 +03:00
parent e3338a1f18
commit e23c5a61a3
Signed by: krd
GPG Key ID: 4D56F87A8BA65FD0
12 changed files with 295 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
jinja2/__pycache__

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
__pycache__
*.py[co]

23
Dockerfile
View File

@ -26,9 +26,14 @@ SHELL [ "/bin/sh", "-ec" ]
COPY /scripts/* /usr/local/sbin/
COPY /extra-scripts/* /usr/local/sbin/
COPY /jinja2/ /usr/local/lib/jinja2/
ENV PYTHONDONTWRITEBYTECODE=''
## Python cache preseed
RUN python3 -m compileall -q -j 2 /usr/local/lib/jinja2/
RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
find "${libpython}/" -mindepth 1 -maxdepth 1 -printf '%P\0' \
| sed -zEn \
@ -45,17 +50,14 @@ RUN libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
python3 -m compileall -q -j 2
## Python cache warmup
RUN python3 -m site > /dev/null ; \
echo > /tmp/f.j2 ; \
jinja.py /tmp/f.j2 ; \
pip-env.sh pip list -v >/dev/null ; \
find "${PYTHON_SITE_PACKAGES}/pip/" -name __pycache__ -exec rm -rf {} +
RUN echo > /tmp/f.j2 ; \
j2-single /tmp/f.j2 ; \
rm -f /tmp/f /tmp/f.j2
## Python cache adjustments
RUN d="@$(date '+%s')" ; \
libpython="${PYTHON_SITE_PACKAGES%/*}" ; \
find "${libpython}/" -name '*.pyc' -exec touch -m -d "$d" {} + ; \
find "${libpython}/" -name __pycache__ -exec touch -m -d "$d" {} +
find /usr/local/lib/ -name '*.pyc' -exec touch -m -d "$d" {} + ; \
find /usr/local/lib/ -name __pycache__ -exec touch -m -d "$d" {} +
## ---
@ -71,7 +73,10 @@ COPY --from=certs /usr/local/share/ca-certificates/ /usr/local/share/ca-certif
## RFC: Python cache
## TODO: reduce load by selecting only __pycache__ directories in either way
# COPY --from=pycache /usr/local/lib/ /usr/local/lib/
COPY --from=pycache /usr/local/lib/ /usr/local/lib/
## already copied by statement above
# COPY /jinja2/ /usr/local/lib/jinja2/
ENV ANGIE_MODULES_DIR=/usr/lib/angie/modules

92
image-entry.d/00-common.envsh
View File

@ -117,55 +117,48 @@ install_userdir() {
fi
}
untemplate_file_envsubst() {
[ -n "$1" ] || return
[ -f "$1" ] || { log_always "file not found: $1" ; return 1 ; }
expand_file_envsubst() {
__r=0
for __src ; do
[ -n "${__src}" ] || continue
[ -n "${NGX_ENVSUBST_SUFFIX:-}" ] || { log "NGX_ENVSUBST_SUFFIX is empty" ; return 1 ; }
__dest="$2"
[ -n "${__dest}" ] || __dest=$(untemplate_path "$1" "${NGX_ENVSUBST_SUFFIX}") || return
if [ -e "${__dest}" ] ; then
log "untemplate_file_envsubst: destination file already exists"
return
if ! [ -f "${__src}" ] ; then
__r=1
log_always "file not found: ${__src}"
continue
fi
[ -d "${__dest%/*}" ] || install_userdir "${__dest%/*}" || return
log "Running envsubst: $1 -> ${__dest}"
envsubst.sh < "$1" > "${__dest}" || return
}
## notes:
## - (OPTIONAL) place own wrapper script as "/usr/local/sbin/jinja.py"
## in order to perform different template processing
untemplate_file_jinja() {
[ -n "$1" ] || return
[ -f "$1" ] || { log_always "file not found: $1" ; return 1 ; }
[ -n "${NGX_JINJA_SUFFIX:-}" ] || { log "NGX_JINJA_SUFFIX is empty" ; return 1 ; }
__dest="$2"
[ -n "${__dest}" ] || __dest=$(untemplate_path "$1" "${NGX_JINJA_SUFFIX}") || return
case "${__src}" in
*.in ) ;;
* )
__r=1
log "expand_file_envsubst: file name extension mismatch: ${__src}"
continue
;;
esac
__dest=$(strip_suffix "${__src}" '.in')
if [ -e "${__dest}" ] ; then
log "untemplate_file_jinja: destination file already exists"
return
__r=1
log "expand_file_envsubst: destination file already exists: ${__dest}"
continue
fi
[ -d "${__dest%/*}" ] || install_userdir "${__dest%/*}" || return
log "Running jinja.py: $1 -> ${__dest}"
jinja.py "$1" "${__dest}" || return
log "Running envsubst: ${__src} -> ${__dest}"
envsubst.sh < "${__src}" > "${__dest}" || __r=1
done
unset __src __dest
return ${__r}
}
untemplate_dir_envsubst() {
[ -n "${NGX_ENVSUBST_SUFFIX:-}" ] || { log "NGX_ENVSUBST_SUFFIX is empty" ; return 1 ; }
expand_file_jinja() {
j2-single "$@" || return $?
}
expand_dir_envsubst() {
__template_list=$(mktemp) || return
find "$@" -follow -type f -name "*${NGX_ENVSUBST_SUFFIX}" \
find "$@" -follow -type f -name '*.in' \
| sort -uV > "${__template_list}"
__have_args="${ENVSUBST_ARGS:+1}"
@ -177,9 +170,10 @@ untemplate_dir_envsubst() {
export ENVSUBST_ARGS
fi
__ret=0
while read -r __orig_file ; do
[ -n "${__orig_file}" ] || continue
untemplate_file_envsubst "${__orig_file}"
expand_file_envsubst "${__orig_file}" || __ret=1
done < "${__template_list}"
unset __orig_file
@ -189,23 +183,25 @@ untemplate_dir_envsubst() {
unset __have_args
rm -f "${__template_list}" ; unset __template_list
return ${__ret}
}
untemplate_dir_jinja() {
[ -n "${NGX_JINJA_SUFFIX:-}" ] || { log "NGX_JINJA_SUFFIX is empty" ; return 1 ; }
expand_dir_jinja() {
__template_list=$(mktemp) || return
find "$@" -follow -type f -name "*${NGX_JINJA_SUFFIX}" \
| sort -uV > "${__template_list}"
find "$@" -follow -type f -name '*.j2' -printf '%p\0' \
| sort -zuV > "${__template_list}"
while read -r __orig_file ; do
[ -n "${__orig_file}" ] || continue
untemplate_file_jinja "${__orig_file}"
done < "${__template_list}"
unset __orig_file
__ret=0
if [ -s "${__template_list}" ] ; then
xargs -0r -n 1000 -a "${__template_list}" \
j2-multi < /dev/null || __ret=1
fi
rm -f "${__template_list}" ; unset __template_list
return ${__ret}
}
remap_path() {

11
image-entry.d/01-defaults.envsh
View File

@ -18,17 +18,6 @@ NGX_CORE_ENV="${NGX_CORE_ENV:-}"
NGX_PROCESS_STATIC=$(gobool_to_int "${NGX_PROCESS_STATIC:-0}" 0)
NGX_ENVSUBST_SUFFIX="${NGX_ENVSUBST_SUFFIX:-.in}"
case "${NGX_ENVSUBST_SUFFIX}" in
.* ) ;;
* ) NGX_ENVSUBST_SUFFIX=".${NGX_ENVSUBST_SUFFIX}" ;;
esac
NGX_JINJA_SUFFIX="${NGX_JINJA_SUFFIX:-.j2}"
case "${NGX_JINJA_SUFFIX}" in
.* ) ;;
* ) NGX_JINJA_SUFFIX=".${NGX_JINJA_SUFFIX}" ;;
esac
set +a
if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then

12
image-entry.d/53-expand-templates.sh
View File

@ -3,7 +3,15 @@ set -ef
. /image-entry.d/00-common.envsh
untemplate_dir_envsubst "${merged_root}"
untemplate_dir_jinja "${merged_root}"
dirs='conf mod modules njs site snip'
[ "${NGX_PROCESS_STATIC}" = 0 ] || dirs="${dirs} static"
for n in ${dirs} ; do
merged_dir="${merged_root}/$n"
[ -d "${merged_dir}" ] || continue
expand_dir_envsubst "${merged_dir}/"
expand_dir_jinja "${merged_dir}/"
done
exit 0

25
jinja2/j2-multi.py Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env python3
import os.path
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import j2common
def main():
if len(sys.argv) < 2:
raise ValueError('not enough arguments (min: 1)')
j2common.init()
ret = 0
for f in sys.argv[1:]:
if not j2common.render_file(f, None, False):
ret = 1
sys.exit(ret)
if __name__ == "__main__":
main()

30
jinja2/j2-single.py Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env python3
import os.path
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import j2common
def main():
if len(sys.argv) < 2:
raise ValueError('not enough arguments (min: 1)')
if len(sys.argv) > 3:
raise ValueError('too many arguments (max: 2)')
if not sys.argv[1]:
raise ValueError('specify input file')
if len(sys.argv) == 3:
if not sys.argv[2]:
raise ValueError('specify output file')
j2common.init()
j2common.render_file(*sys.argv[1:])
sys.exit(0)
if __name__ == "__main__":
main()

156
jinja2/j2common.py Executable file
View File

@ -0,0 +1,156 @@
#!/usr/bin/env python3
import importlib
import json
import os
import os.path
import sys
import yaml
import jinja2
ME = sys.argv[0]
J2_MODULES_DEFAULT = 'os os.path sys netaddr psutil re wcmatch'
J2_SUFFIX = '.j2'
J2_CFG_PATHS = [
'/angie/j2cfg',
'/etc/angie/j2cfg',
]
J2_CFG_EXTS = [
'yml',
'yaml',
'json',
]
J2_SEARCH_PATH = [
'/etc/angie',
'/run/angie',
'/',
]
J2_EXT_LIST = [
'jinja2.ext.do',
'jinja2.ext.loopcontrols',
]
J2_MODULES = sorted(set(
os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ')
))
J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '')
J2_KWARGS = {}
def merge_dict_from_file(filename):
if (not filename) or (filename == ''):
return False
if not os.path.exists(filename):
return False
if not os.path.isfile(filename):
print(
f'{ME}: not a file, skipping: {filename}',
file=sys.stderr)
return False
if filename.endswith('.yml') or filename.endswith('.yaml'):
with open(filename, mode='r', encoding='utf-8') as fx:
x = yaml.safe_load(fx)
J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
return True
if filename.endswith('.json'):
with open(filename, mode='r', encoding='utf-8') as fx:
x = json.load(fx)
J2_KWARGS['cfg'] = J2_KWARGS['cfg'] | x
return True
print(
f'{ME}: non-recognized name extension: {filename}',
file=sys.stderr)
return False
def merge_dict_default():
for base in J2_CFG_PATHS:
for full in [base + '.' + ext for ext in J2_CFG_EXTS]:
if merge_dict_from_file(full):
break
continue
def render_error(msg, fail=True) -> bool:
if fail:
raise ValueError(msg)
print(f'{ME}: {msg}', file=sys.stderr)
return False
def render_file(file_in, file_out=None, fail=True):
if (not file_in) or (file_in == ''):
return render_error(
'argument "file_in" is empty',
fail)
if not os.path.exists(file_in):
return render_error(
f'file is missing: {file_in}',
fail)
if not os.path.isfile(file_in):
return render_error(
f'not a file: {file_in}',
fail)
if file_out:
if str(file_in) == str(file_out):
return render_error(
f'unable to process template inplace: {file_in}',
fail)
f_out = file_out
else:
if not file_in.endswith(J2_SUFFIX):
return render_error(
f'input file name extension mismatch: {file_in}',
fail)
f_out = os.path.splitext(file_in)[0]
if os.path.exists(f_out):
return render_error(
f'output file already exists: {f_out}',
fail)
dirs = J2_SEARCH_PATH.copy()
for d in [os.path.dirname(file_in), os.getcwd()]:
if d not in dirs:
dirs.insert(0, d)
j2_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader(
d,
encoding='utf-8',
followlinks=True,
) for d in dirs
])
j2_environ = jinja2.Environment(
loader=j2_loader,
extensions=J2_EXT_LIST,
)
j2_template = j2_environ.get_template(file_in)
j2_stream = j2_template.stream(**J2_KWARGS)
j2_stream.disable_buffering()
j2_stream.dump(f_out, encoding='utf-8')
return True
def init():
global J2_KWARGS
for m in J2_MODULES:
J2_KWARGS[m] = importlib.import_module(m)
J2_KWARGS['env'] = os.environ
J2_KWARGS['cfg'] = {}
if J2_CONFIG != '':
if os.path.isfile(J2_CONFIG):
merge_dict_from_file(J2_CONFIG)
else:
print(
f'{ME}: J2_CONFIG does not exist, skipping: {J2_CONFIG}',
file=sys.stderr
)
merge_dict_default()
else:
merge_dict_default()

6
scripts/j2-multi Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
[ "${IEP_VERBOSE}" = 0 ] || {
echo "Running ${0##*/}:" >&2
printf ' - %s\n' "$@" >&2
}
exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"

5
scripts/j2-single Executable file
View File

@ -0,0 +1,5 @@
#!/bin/sh
[ "${IEP_VERBOSE}" = 0 ] || {
echo "Running ${0##*/}: $*" >&2
}
exec python3 "/usr/local/lib/jinja2/${0##*/}.py" "$@"

119
scripts/jinja.py
View File

@ -1,119 +0,0 @@
#!/usr/bin/env python3
import importlib
import json
import os
import os.path
import sys
import jinja2
import yaml
J2_MODULES_DEFAULT = 'os os.path sys netaddr psutil re wcmatch'
J2_SUFFIX_DEFAULT = '.j2'
J2_CFG_PATHS = [
'/angie/jinja',
'/etc/angie/jinja',
]
J2_CFG_EXTS = [
'yml',
'yaml',
'json',
]
J2_SEARCH_PATH = [
'.',
'/etc/angie',
]
J2_EXT_LIST = [
'jinja2.ext.do',
'jinja2.ext.loopcontrols',
]
J2_MODULES = sorted(set(os.getenv('NGX_JINJA_MODULES', J2_MODULES_DEFAULT).split(sep=' ')))
ME = sys.argv[0]
J2_SUFFIX = os.getenv('NGX_JINJA_SUFFIX', J2_SUFFIX_DEFAULT)
if J2_SUFFIX == '':
raise ValueError('NGX_JINJA_SUFFIX is empty')
if not J2_SUFFIX.startswith('.'):
raise ValueError('NGX_JINJA_SUFFIX does not start with dot (".")')
J2_CONFIG = os.getenv('NGX_JINJA_CONFIG', '')
if (J2_CONFIG != '') and (not os.path.exists(J2_CONFIG)):
print(f'{ME}: config does not exist, skipping: {J2_CONFIG}', file=sys.stderr)
if len(sys.argv) < 2:
raise ValueError('not enough arguments (needed: 2)')
if len(sys.argv) > 3:
raise ValueError('too many arguments (needed: 2)')
if not sys.argv[1]:
raise ValueError('specify input file')
input_file = sys.argv[1]
if not os.path.exists(input_file):
raise ValueError('input file does not exist')
if len(sys.argv) == 3:
if not sys.argv[2]:
raise ValueError('specify output file')
output_file = sys.argv[2]
else:
output_file, ext = os.path.splitext(input_file)
if ext != J2_SUFFIX:
raise ValueError(f'input file name extension mismatch (not a "{J2_SUFFIX}")')
if input_file == output_file:
raise ValueError('unable to process template inplace')
kwargs = {}
for m in J2_MODULES:
kwargs[m] = importlib.import_module(m)
kwargs['env'] = os.environ
kwargs['cfg'] = {}
def merge_dict_from_file(filename):
if not filename:
return False
if not isinstance(filename, str):
return False
if filename == '':
return False
if not os.path.exists(filename):
return False
if not os.path.isfile(filename):
print(f'{ME}: not a file, skipping: {filename}', file=sys.stderr)
return False
if filename.endswith('.yml') or filename.endswith('.yaml'):
with open(filename, mode='r', encoding='utf-8') as fx:
x = yaml.safe_load(fx)
kwargs['cfg'] = kwargs['cfg'] | x
return True
if filename.endswith('.json'):
with open(filename, mode='r', encoding='utf-8') as fx:
x = json.load(fx)
kwargs['cfg'] = kwargs['cfg'] | x
return True
print(f'{ME}: non-recognized name extension: {filename}', file=sys.stderr)
return False
if (J2_CONFIG != '') and (os.path.isfile(J2_CONFIG)):
merge_dict_from_file(J2_CONFIG)
else:
for base in J2_CFG_PATHS:
for full in [ base + '.' + ext for ext in J2_CFG_EXTS ]:
if merge_dict_from_file(full):
break
continue
j2_loader = jinja2.FileSystemLoader(J2_SEARCH_PATH, followlinks=True)
j2_environ = jinja2.Environment(loader=j2_loader, extensions=J2_EXT_LIST)
j2_template = j2_environ.get_template(input_file)
j2_stream = j2_template.stream(**kwargs)
j2_stream.disable_buffering()
j2_stream.dump(output_file, encoding='utf-8')