treewide: improve template expansion

also: support *.toml as extra configuration dictionaries

image-entry.d/00-common.envsh

@@ -82,99 +82,6 @@ user_install() {
 	fi
 }
 
-expand_file_envsubst() {
-	local __ret __src __dst
-
-	__ret=0
-	for __src ; do
-		[ -n "${__src}" ] || continue
-
-		if ! [ -f "${__src}" ] ; then
-			__ret=1
-			log_always "file not found: ${__src}"
-			continue
-		fi
-
-		case "${__src}" in
-		*.in ) ;;
-		* )
-			__ret=1
-			log "expand_file_envsubst: file name extension mismatch: ${__src}"
-			continue
-		;;
-		esac
-
-		__dst=$(strip_suffix "${__src}" '.in')
-		if [ -e "${__dst}" ] ; then
-			__ret=1
-			log "expand_file_envsubst: destination file already exists: ${__dst}"
-			continue
-		fi
-
-		log "Running envsubst: ${__src} -> ${__dst}"
-		envsubst.sh < "${__src}" > "${__dst}" || __ret=1
-	done
-	return ${__ret}
-}
-
-expand_file_j2cfg() {
-	j2cfg-single "$@" || return $?
-}
-
-expand_dir_envsubst() {
-	local __template_list __have_args __ret __orig_file
-
-	__template_list=$(mktemp) || return
-
-	find "$@" -follow -name '*.in' -type f \
-	| sort -uV > "${__template_list}"
-
-	__ret=0
-	if [ -s "${__template_list}" ] ; then
-		__have_args="${ENVSUBST_ARGS:+1}"
-		if [ -z "${__have_args}" ] ; then
-			## optimize envsubst.sh invocation by caching argument list
-			## ref: envsubst.sh
-			ENVSUBST_ARGS=$(mktemp) || return
-			envsubst-args.sh > "${ENVSUBST_ARGS}"
-			export ENVSUBST_ARGS
-		fi
-
-		while read -r __orig_file ; do
-			[ -n "${__orig_file}" ] || continue
-			expand_file_envsubst "${__orig_file}" || __ret=1
-		done < "${__template_list}"
-
-		if [ -z "${__have_args}" ] ; then
-			rm -f "${ENVSUBST_ARGS}" ; unset ENVSUBST_ARGS
-		fi
-		unset __have_args
-	fi
-
-	rm -f "${__template_list}" ; unset __template_list
-
-	return ${__ret}
-}
-
-expand_dir_j2cfg() {
-	local __template_list __ret
-
-	__template_list=$(mktemp) || return
-
-	find "$@" -follow -name '*.j2' -type f -printf '%p\0' \
-	| sort -zuV > "${__template_list}"
-
-	__ret=0
-	if [ -s "${__template_list}" ] ; then
-		xargs -0r -n 1000 -a "${__template_list}" \
-		j2cfg-multi < /dev/null || __ret=1
-	fi
-
-	rm -f "${__template_list}" ; unset __template_list
-
-	return ${__ret}
-}
-
 is_builtin_module() {
 	[ -n "${1:-}" ] || return 1
 	[ -n "${2:-}" ] || return 1

image-entry.d/01-defaults.envsh

@@ -1,5 +1,13 @@
 #!/bin/sh
 
+## if IEP_DEBUG is not set, allow scripts to delete source templates
+## (they are likely not needed anymore)
+if [ "${IEP_DEBUG}" = 1 ] ; then
+	unset ENVSUBST_UNLINK_SRC J2CFG_UNLINK_SRC
+else
+	export ENVSUBST_UNLINK_SRC=1 J2CFG_UNLINK_SRC=1
+fi
+
 unset NGX_DEBUG
 NGX_DEBUG=$(/usr/sbin/angie --build-env 2>&1 | mawk '$1=="DEBUG:" {print $2;exit;}')
 NGX_DEBUG="${NGX_DEBUG:-0}"

image-entry.d/70-merge-dirs.sh

@@ -16,6 +16,9 @@ overlaydirs --merge "${target_root}" /etc/angie.dist /etc/angie /angie "${fake_d
 ## fixup after merge
 for n in ${persist_dirs} ; do rm -f "${target_root}/$n" ; done
 rm -rf "${fake_dir}"
+if [ -d "${target_root}/mod" ] ; then
+	find "${target_root}/mod/" -follow -name '.*.preseed' -type f -exec rm -f {} +
+fi
 
 if [ "${NGX_HTTP_STATIC_MERGE}" = 0 ] ; then
 	src0=/etc/angie.dist/static

image-entry.d/73-expand-templates.sh

@@ -29,9 +29,6 @@ expand_error() {
 	expand_error_delim
 }
 
-set +e
-## NB: j2cfg/ and static/ are handled separately
-merge_dirs=$(find ./ -follow -mindepth 1 -maxdepth 1 -type d -printf '%P/\n' | grep -Fxv -e j2cfg/ -e static/ | sort -uV)
 [ "${NGX_STRICT_LOAD}" = 0 ] || set -e
 
 unset ENVSUBST_ARGS
@@ -40,9 +37,7 @@ envsubst-args.sh > "${ENVSUBST_ARGS}"
 export ENVSUBST_ARGS
 
 ## envsubst is simple and fast
-## expand j2cfg/ first, then other directories
-expand_dir_envsubst j2cfg/ || expand_error
-expand_dir_envsubst ${merge_dirs} || expand_error
+envsubst-dirs j2cfg/ || expand_error
 
 ## j2cfg is more complex
 
@@ -53,39 +48,53 @@ J2CFG_SEARCH_PATH="${target_root}"
 set +a
 
 ## expand j2cfg/ first
-expand_dir_j2cfg j2cfg/ || expand_error
+j2cfg-dirs j2cfg/ || expand_error
 
 ## dump [merged] j2cfg config
 j2cfg_dump="${volume_root}/diag.j2cfg.yml"
-j2cfg-dump > "${j2cfg_dump}" || expand_error
+j2cfg-dump-yml > "${j2cfg_dump}" || expand_error
 export J2CFG_CONFIG="${j2cfg_dump}"
 
 ## expand other directories
-expand_dir_j2cfg ${merge_dirs} || expand_error
+## NB: j2cfg/ and static/ are handled separately
+merge_dirs=$(mktemp)
+{
+	set +e
+	find ./ -follow -mindepth 1 -maxdepth 1 -type d -printf '%P/\0' \
+	| grep -zFxv -e j2cfg/ -e static/ | sort -zuV
+} > "${merge_dirs}"
+
+xargs -0r -n 1000 -a "${merge_dirs}" \
+envsubst-dirs < /dev/null || expand_error
+
+xargs -0r -n 1000 -a "${merge_dirs}" \
+j2cfg-dirs < /dev/null || expand_error
+
+rm -f "${merge_dirs}" ; unset merge_dirs
 
 ## expand static/
-## remove template sources in order to avoid leaking sensitive data
+## NB: template sources are removed unless IEP_DEBUG is set!
 if [ "${NGX_HTTP_STATIC_TEMPLATE}" = 1 ] ; then
+
 	template_list=$(mktemp)
 
-	find static/ -follow -name '*.in' -type f \
+	find static/ -follow -name '*.in' -type f -printf '%p\0' \
 	| {
 		set +e
 		if [ -n "${NGX_STATIC_EXCLUDE_REGEX:-}" ] ; then
-			grep -Ev -e "${NGX_STATIC_EXCLUDE_REGEX}"
+			grep -zEv -e "${NGX_STATIC_EXCLUDE_REGEX}"
 		elif [ -n "${NGX_STATIC_INCLUDE_REGEX:-}" ] ; then
-			grep -E -e "${NGX_STATIC_INCLUDE_REGEX}"
+			grep -zE -e "${NGX_STATIC_INCLUDE_REGEX}"
 		else
 			cat
 		fi
 	} \
-	| sort -uV > "${template_list}"
+	| sort -zuV > "${template_list}"
 
-	while read -r src ; do
-		[ -n "${src}" ] || continue
-		expand_file_envsubst "${src}" || expand_error
-		rm -fv "${src}"
-	done < "${template_list}"
+	if [ -s "${template_list}" ] ; then
+		xargs -0r -n 1000 -a "${template_list}" \
+		envsubst-multi < /dev/null || expand_error
+	fi
 
 	find static/ -follow -name '*.j2' -type f -printf '%p\0' \
 	| {
@@ -103,9 +112,6 @@ if [ "${NGX_HTTP_STATIC_TEMPLATE}" = 1 ] ; then
 	if [ -s "${template_list}" ] ; then
 		xargs -0r -n 1000 -a "${template_list}" \
 		j2cfg-multi < /dev/null || expand_error
-
-		xargs -0r -n 1000 -a "${template_list}" \
-		rm -fv < /dev/null
 	fi
 
 	rm -f "${template_list}"

image-entry.d/99-cleanup-env.envsh

@@ -47,7 +47,7 @@ $(
 	| xargs -0r printf '%q\n' \
 	| {
 		## retain variables defined in ".core_worker_env" configuration key
-		## (if it was specified somewhere in dictionaries - either yaml or json)
+		## (if it was specified somewhere in dictionaries - either yaml, toml or json)
 		f="${target_root}/autoconf/core-worker-env.txt"
 		[ -s "$f" ] || exec cat
 		grep -Fxv -f "$f"