refactoring

Dockerfile

@@ -174,6 +174,7 @@ RUN x='angie-builtin-modules.sh' ; \
 ## misc tools
 RUN apt-install.sh \
       brotli \
+      curl \
       zstd \
     ; \
     apt-clean.sh

angie/.none.conf

@@ -1,3 +0,0 @@
-daemon off;
-master_process off;
-events {}

angie/autoconf.dist/http-alt-svc.conf.j2

@@ -0,0 +1,12 @@
+{#- prologue -#}
+{%- set extra_proto = ['v3', 'v2'] -%}
+{%- set confload = ( env.NGX_HTTP_CONFLOAD or '' ) | str_split_to_list -%}
+{%- set proto = confload | list_intersect(extra_proto) -%}
+{#- ALPN mapping -#}
+{%- set proto = proto | re_sub('^v2$', 'h2=":443"; ma=3600') -%}
+{%- set proto = proto | re_sub('^v3$', 'h3=":443"; ma=3600') -%}
+{#- main part -#}
+{%- if proto %}
+{#- TODO: precise quotation #}
+add_header  Alt-Svc  {{ (proto | join(', ')).__repr__() }};
+{%- endif %}

angie/autoconf.dist/http-request-headers-basic.conf.j2

@@ -0,0 +1,26 @@
+map $http_upgrade
+    $req_connection
+{
+    default  upgrade;
+    ""       "";
+}
+
+map $http_user_agent
+    $req_user_agent
+{
+    default  $http_user_agent;
+{%- if env.NGX_HTTP_FAKE_UA %}
+    ## merely fake
+    ""  {{ env.NGX_HTTP_FAKE_UA.__repr__() }};
+{%- else %}
+    ""  "Angie/$angie_version";
+{%- endif %}
+}
+
+map $http_accept
+    $req_accept
+{
+    volatile;
+    default  $http_accept;
+    ""       "*/*";
+}

angie/autoconf.dist/http-request-headers-forwarded.conf

@@ -0,0 +1,27 @@
+## ref:
+## - https://www.digitalocean.com/community/tools/nginx?domains.0.reverseProxy.reverseProxy=true
+map $remote_addr
+    $proxy_forwarded_elem
+{
+    ## IPv4 addresses can be sent as-is
+    ~^[0-9.]+$         "for=$remote_addr";
+    ## IPv6 addresses need to be bracketed and quoted
+    ~^[0-9A-Fa-f:.]+$  "for=\"[$remote_addr]\"";
+    ## Unix domain socket names cannot be represented in RFC 7239 syntax
+    default            "for=unknown";
+}
+
+## ref:
+## - https://www.digitalocean.com/community/tools/nginx?domains.0.reverseProxy.reverseProxy=true
+## - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
+map $http_forwarded
+    $proxy_add_forwarded
+{
+    volatile;
+
+    ## if the incoming Forwarded header is syntactically valid, append to it
+    "~^(,[ \\t]*)*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*([ \\t]*,([ \\t]*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*)?)*$" "$http_forwarded, $proxy_forwarded_elem";
+
+    ## otherwise, replace it
+    default  "$proxy_forwarded_elem";
+}

angie/conf.dist/fastcgi/cache-bypass.conf.j2

@@ -0,0 +1,15 @@
+{#- TODO: precise quotation -#}
+{%- set cache_bypass = j2cfg.fastcgi_cache_bypass or j2cfg.cache_bypass or [] -%}
+{%- if cache_bypass -%}
+## disable (response) cache under following conditions
+fastcgi_cache_bypass
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+fastcgi_no_cache
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+{%- endif -%}

angie/conf.dist/fastcgi/headers.conf.j2

@@ -1,13 +1,13 @@
 ## hide/remove request headers
-{%- set req_hdr_list = j2cfg.fastcgi_remove_request_headers or j2cfg.remove_request_headers or [] -%}
+{%- set req_hdr_dict = j2cfg.fastcgi_request_headers or j2cfg.request_headers or {} -%}
-{%- set req_hdr_list = req_hdr_list | any_to_str_list | as_cgi_header -%}
+{%- for h, v in req_hdr_dict.items() %}
-{%- for h in req_hdr_list %}
+{#- TODO: precise quotation #}
-fastcgi_param  {{ h }}  "";
+fastcgi_param  {{ h | as_cgi_header }}  {{ v.__repr__() }};
 {%- endfor %}
 
 ## hide response headers
-{%- set resp_hdr_list = j2cfg.fastcgi_remove_response_headers or j2cfg.remove_response_headers or [] -%}
+{%- set resp_hdr_dict = j2cfg.fastcgi_response_headers or j2cfg.response_headers or {} -%}
-{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%}
 {%- for h in resp_hdr_list %}
 fastcgi_hide_header  {{ h }};
 {%- endfor %}

angie/conf.dist/grpc/headers.conf.j2

@@ -1,13 +1,13 @@
 ## hide/remove request headers
-{%- set req_hdr_list = j2cfg.grpc_remove_request_headers or j2cfg.remove_request_headers or [] -%}
+{%- set req_hdr_dict = j2cfg.grpc_request_headers or j2cfg.request_headers or {} -%}
-{%- set req_hdr_list = req_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- for h, v in req_hdr_dict.items() %}
-{%- for h in req_hdr_list %}
+{#- TODO: precise quotation #}
-grpc_set_header  {{ h }}  "";
+grpc_set_header  {{ h }}  {{ v.__repr__() }};
 {%- endfor %}
 
 ## hide response headers
-{%- set resp_hdr_list = j2cfg.grpc_remove_response_headers or j2cfg.remove_response_headers or [] -%}
+{%- set resp_hdr_dict = j2cfg.grpc_response_headers or j2cfg.response_headers or {} -%}
-{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%}
 {%- for h in resp_hdr_list %}
 grpc_hide_header  {{ h }};
 {%- endfor %}

angie/conf.dist/http-grpc.conf

@@ -1,4 +1 @@
-## this should be enabled explicitly to avoid config mess
-# include conf.d/http-v2.conf;
-
 include conf.d/grpc/*.conf;

angie/conf.dist/http-proxy.conf

@@ -0,0 +1 @@
+include conf.d/proxy/*.conf;

angie/conf.dist/http-quic-gso.conf

@@ -0,0 +1,3 @@
+quic_gso on;
+
+proxy_quic_gso on;

angie/conf.dist/http-response-headers.conf.j2

@@ -1,6 +1,6 @@
 ## add response headers
-{%- set resp_hdr_list = ( j2cfg.add_response_headers or {} ) -%}
+{%- set resp_hdr_dict = j2cfg.response_headers or {} -%}
-{%- for h, v in resp_hdr_list.items() %}
+{%- for h, v in resp_hdr_dict.items() %}
 {#- TODO: precise quotation #}
 add_header  {{ h }}  {{ v.__repr__() }};
 {%- endfor %}

angie/conf.dist/http-v2.conf

@@ -1,2 +1,2 @@
-http2_chunk_size 16k;
+include conf.d/http2/*.conf;
 http2 on;

angie/conf.dist/http-v3.conf

@@ -0,0 +1,2 @@
+include conf.d/http3/*.conf;
+http3 on;

angie/conf.dist/http2/param.conf

@@ -0,0 +1,2 @@
+http2_chunk_size 16k;
+http2_body_preread_size 64k;

angie/conf.dist/http3/param.conf

@@ -0,0 +1,7 @@
+http3_max_concurrent_streams 128; #default
+http3_stream_buffer_size 64k; #default
+quic_active_connection_id_limit 3;
+
+proxy_http3_max_concurrent_streams 128; #default
+proxy_http3_stream_buffer_size 64k; #default
+proxy_quic_active_connection_id_limit 3;

angie/conf.dist/proxy/buffers.conf

@@ -0,0 +1,4 @@
+proxy_buffers           16  16k;
+proxy_buffer_size           16k;
+proxy_busy_buffers_size     32k;
+proxy_temp_file_write_size  32k;

angie/conf.dist/proxy/cache-bypass.conf.j2

@@ -0,0 +1,15 @@
+{#- TODO: precise quotation -#}
+{%- set cache_bypass = j2cfg.proxy_cache_bypass or j2cfg.cache_bypass or [] -%}
+{%- if cache_bypass -%}
+## disable (response) cache under following conditions
+proxy_cache_bypass
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+proxy_no_cache
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+{%- endif -%}

angie/conf.dist/proxy/headers.conf.j2

@@ -1,13 +1,13 @@
 ## hide/remove request headers
-{%- set req_hdr_list = j2cfg.proxy_remove_request_headers or j2cfg.remove_request_headers or [] -%}
+{%- set req_hdr_dict = j2cfg.proxy_request_headers or j2cfg.request_headers or {} -%}
-{%- set req_hdr_list = req_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- for h, v in req_hdr_dict.items() %}
-{%- for h in req_hdr_list %}
+{#- TODO: precise quotation #}
-proxy_set_header  {{ h }}  "";
+proxy_set_header  {{ h }}  {{ v.__repr__() }};
 {%- endfor %}
 
 ## hide response headers
-{%- set resp_hdr_list = j2cfg.proxy_remove_response_headers or j2cfg.remove_response_headers or [] -%}
+{%- set resp_hdr_dict = j2cfg.proxy_response_headers or j2cfg.response_headers or {} -%}
-{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%}
 {%- for h in resp_hdr_list %}
 proxy_hide_header  {{ h }};
 {%- endfor %}

angie/conf.dist/proxy/version.conf

@@ -0,0 +1 @@
+proxy_http_version 1.1;

angie/conf.dist/scgi/cache-bypass.conf.j2

@@ -0,0 +1,15 @@
+{#- TODO: precise quotation -#}
+{%- set cache_bypass = j2cfg.scgi_cache_bypass or j2cfg.cache_bypass or [] -%}
+{%- if cache_bypass -%}
+## disable (response) cache under following conditions
+scgi_cache_bypass
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+scgi_no_cache
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+{%- endif -%}

angie/conf.dist/scgi/headers.conf.j2

@@ -1,13 +1,13 @@
 ## hide/remove request headers
-{%- set req_hdr_list = j2cfg.scgi_remove_request_headers or j2cfg.remove_request_headers or [] -%}
+{%- set req_hdr_dict = j2cfg.scgi_request_headers or j2cfg.request_headers or {} -%}
-{%- set req_hdr_list = req_hdr_list | any_to_str_list | as_cgi_header -%}
+{%- for h, v in req_hdr_dict.items() %}
-{%- for h in req_hdr_list %}
+{#- TODO: precise quotation #}
-scgi_param  {{ h }}  "";
+scgi_param  {{ h | as_cgi_header }}  {{ v.__repr__() }};
 {%- endfor %}
 
 ## hide response headers
-{%- set resp_hdr_list = j2cfg.scgi_remove_response_headers or j2cfg.remove_response_headers or [] -%}
+{%- set resp_hdr_dict = j2cfg.scgi_response_headers or j2cfg.response_headers or {} -%}
-{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%}
 {%- for h in resp_hdr_list %}
 scgi_hide_header  {{ h }};
 {%- endfor %}

angie/conf.dist/uwsgi/cache-bypass.conf.j2

@@ -0,0 +1,15 @@
+{#- TODO: precise quotation -#}
+{%- set cache_bypass = j2cfg.uwsgi_cache_bypass or j2cfg.cache_bypass or [] -%}
+{%- if cache_bypass -%}
+## disable (response) cache under following conditions
+uwsgi_cache_bypass
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+uwsgi_no_cache
+{%- for v in cache_bypass %}
+    {{ v.__repr__() }}
+{%- endfor %}
+;
+{%- endif -%}

angie/conf.dist/uwsgi/headers.conf.j2

@@ -1,13 +1,13 @@
 ## hide/remove request headers
-{%- set req_hdr_list = j2cfg.uwsgi_remove_request_headers or j2cfg.remove_request_headers or [] -%}
+{%- set req_hdr_dict = j2cfg.uwsgi_request_headers or j2cfg.request_headers or {} -%}
-{%- set req_hdr_list = req_hdr_list | any_to_str_list | as_cgi_header -%}
+{%- for h, v in req_hdr_dict.items() %}
-{%- for h in req_hdr_list %}
+{#- TODO: precise quotation #}
-uwsgi_param  {{ h }}  "";
+uwsgi_param  {{ h | as_cgi_header }}  {{ v.__repr__() }};
 {%- endfor %}
 
 ## hide response headers
-{%- set resp_hdr_list = j2cfg.uwsgi_remove_response_headers or j2cfg.remove_response_headers or [] -%}
+{%- set resp_hdr_dict = j2cfg.uwsgi_response_headers or j2cfg.response_headers or {} -%}
-{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%}
+{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%}
 {%- for h in resp_hdr_list %}
 uwsgi_hide_header  {{ h }};
 {%- endfor %}

angie/j2cfg.dist/add-response-headers.yml

@@ -1,11 +0,0 @@
-add_response_headers:
-  Access-Control-Allow-Origin: "*"
-  Access-Control-Allow-Headers: "Origin, X-Requested-With, Content-Type, Accept, Authorization"
-  Access-Control-Allow-Methods: "GET, HEAD, POST, PUT, DELETE, OPTIONS"
-  Content-Security-Policy: "default-src 'self' http: https: ws: wss: data: blob: 'unsafe-inline' 'unsafe-eval' ; frame-ancestors 'self';"
-  Permissions-Policy: "microphone=(), camera=(), geolocation=(), interest-cohort=()"
-  Referrer-Policy: "no-referrer-when-downgrade"
-  Strict-Transport-Security: "max-age=31536000; includeSubDomains; preload"
-  X-Content-Type-Options: "nosniff"
-  X-Frame-Options: "SAMEORIGIN"
-  X-XSS-Protection: "1; mode=block"

angie/j2cfg.dist/cache-bypass.yml

@@ -0,0 +1,4 @@
+cache_bypass:
+- '$http_authorization'
+- '$http_pragma'
+- '$http_upgrade'

angie/j2cfg.dist/core-worker-env.txt.j2

@@ -4,6 +4,6 @@
 {%- set c_vars_passthrough = c_env | dict_empty_keys -%}
 {%- set vars_passthrough = ((env_passthrough | list_diff(c_vars)) + c_vars_passthrough) | uniq | list_intersect(env | dict_keys) -%}
 {#- main part -#}
-{%- for k in vars_passthrough -%}
+{%- for k in vars_passthrough %}
 {{ k }}
-{% endfor -%}
+{%- endfor %}

angie/j2cfg.dist/headers-request-transparent.yml.j2

@@ -0,0 +1,12 @@
+{% if env.NGX_HTTP_TRANSPARENT_PROXY == '0' %}
+request_headers:
+  Host: '$proxy_host'
+  X-Real-IP: '$remote_addr'
+  ## '$proxy_add_forwarded' is defined in /angie/autoconf.dist/http-request-headers-forwarded.conf
+  Forwarded: '$proxy_add_forwarded'
+{% elif env.NGX_HTTP_TRANSPARENT_PROXY == '1' %}
+request_headers:
+  Host: '$host'
+  X-Real-IP: ''
+  Forwarded: ''
+{% endif %}

angie/j2cfg.dist/headers-request-x-forwarded.yml.j2

@@ -0,0 +1,13 @@
+{% if env.NGX_HTTP_X_FORWARDED == 'pass' %}
+request_headers:
+  X-Forwarded-Proto: '$scheme'
+  X-Forwarded-Host: '$host'
+  X-Forwarded-Port: '$server_port'
+  X-Forwarded-For: '$proxy_add_x_forwarded_for'
+{% elif env.NGX_HTTP_X_FORWARDED == 'remove' %}
+request_headers:
+  X-Forwarded-Proto: ''
+  X-Forwarded-Host: ''
+  X-Forwarded-Port: ''
+  X-Forwarded-For: ''
+{% endif %}

angie/j2cfg.dist/headers-request.yml

@@ -0,0 +1,11 @@
+request_headers:
+  ## do not pass Accept-Encoding to backend
+  Accept-Encoding: ""
+  ## '$req_accept' is defined in /angie/autoconf.dist/http-request-headers-basic.conf.j2
+  Accept: '$req_accept'
+  ## '$req_connection' is defined in /angie/autoconf.dist/http-request-headers-basic.conf.j2
+  Connection: '$req_connection'
+  Upgrade: '$http_upgrade'
+  Early-Data: '$ssl_early_data'
+  ## '$req_user_agent' is defined in /angie/autoconf.dist/http-request-headers-basic.conf.j2
+  User-Agent: '$req_user_agent'

angie/j2cfg.dist/headers-response.yml

@@ -0,0 +1,7 @@
+response_headers:
+  Permissions-Policy: "microphone=(), camera=(), geolocation=(), interest-cohort=()"
+  Referrer-Policy: "no-referrer-when-downgrade"
+  Strict-Transport-Security: "max-age=15724800; includeSubDomains; preload"
+  X-Content-Type-Options: "nosniff"
+  X-Frame-Options: "SAMEORIGIN"
+  X-XSS-Protection: "1; mode=block"

angie/j2cfg.dist/remove-request-headers.yml

@@ -1,3 +0,0 @@
-remove_request_headers:
-  ## do not pass Accept-Encoding to backend
-- Accept-Encoding

angie/j2cfg.dist/remove-response-headers.yml

@@ -1,12 +0,0 @@
-remove_response_headers:
-- Access-Control-Allow-Headers
-- Access-Control-Allow-Methods
-- Access-Control-Allow-Origin
-- Content-Security-Policy
-- Permissions-Policy
-- Referrer-Policy
-- Strict-Transport-Security
-- Vary
-- X-Content-Type-Options
-- X-Frame-Options
-- X-XSS-Protection

angie/modsecurity.dist/rules.conf

@@ -10,7 +10,7 @@ Include modsecurity.conf
 #      w=$(mktemp -d) ; : "${w:?}"
 #      cd "$w/"
 #      tarball="coreruleset.tar.gz"
-#      /usr/lib/apt/apt-helper download-file "${uri}" "${tarball}"
+#      curl -Lo "${tarball}" "${uri}"
 #      mkdir coreruleset
 #      tar -C ./coreruleset --strip-components=1 -xf "${tarball}"
 #      rm -f "${tarball}" ; unset tarball

angie/snip.dist/disable-compression.j2

@@ -1,8 +1,8 @@
 {#- safe to specify all the time -#}
 gzip off;
+{%- set extra_comp_modules = ['brotli', 'zstd'] -%}
 {%- set modules = ( env.NGX_HTTP_MODULES or '' ) | str_split_to_list -%}
-{%- for ext_comp in ['brotli', 'zstd'] %}
+{%- set comp_modules = modules | list_intersect(extra_comp_modules) | sort -%}
-{%- if ext_comp in modules %}
+{%- for comp in comp_modules %}
-{{ ext_comp }} off;
+{{ comp }} off;
-{%- endif %}
 {%- endfor %}

image-entry.d/01-defaults.envsh

@@ -19,7 +19,7 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then
 fi
 
 unset default_dirs_merge default_dirs_link
-default_dirs_merge='autoconf conf j2cfg mod modules site snip'
+default_dirs_merge='autoconf conf mod modules site snip'
 default_dirs_link=''
 
 if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then

image-entry.d/21-http-modules.envsh

@@ -3,6 +3,11 @@
 if [ "${NGX_HTTP}" = 0 ] ; then
 	unset NGX_HTTP_NO_PROXY NGX_HTTP_WITH_MODSECURITY
 else
+	NGX_HTTP_NO_PROXY=$(gobool_to_int "${NGX_HTTP_NO_PROXY:-0}" 0)
+	if [ "${NGX_HTTP_NO_PROXY}" = 0 ] ; then
+		NGX_HTTP_CONFLOAD=$(append_list "${NGX_HTTP_CONFLOAD}" proxy)
+	fi
+
 	unset http_modules http_confload
 	http_modules=
 	http_confload="${NGX_HTTP_CONFLOAD:-}"
@@ -56,6 +61,19 @@ else
 	done
 	unset i
 
+	## grpc depends on http/2
+	if list_have_item "${NGX_HTTP_CONFLOAD}" grpc ; then
+		unset want_http2
+		want_http2=0
+		if ! list_have_item "${NGX_HTTP_CONFLOAD}" v2 ; then
+			want_http2=1
+		fi
+		if [ "${want_http2}" = 1 ] ; then
+			NGX_HTTP_CONFLOAD=$(append_list "${NGX_HTTP_CONFLOAD}" v2)
+		fi
+		unset want_http2
+	fi
+
 	set -a
 	NGX_HTTP_MODULES="${http_modules}"
 	NGX_HTTP_CONFLOAD=$(sort_dedup_list "${http_confload}")

image-entry.d/23-http-forward-headers.envsh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+if [ "${NGX_HTTP}" = 0 ] ; then
+	unset NGX_HTTP_TRANSPARENT_PROXY NGX_HTTP_FAKE_UA NGX_HTTP_X_FORWARDED
+else
+	unset _NGX_HTTP_FAKE_UA _NGX_HTTP_X_FORWARDED
+	## here should be SANE defaults (!)
+	_NGX_HTTP_FAKE_UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'
+	_NGX_HTTP_X_FORWARDED=pass
+
+	NGX_HTTP_TRANSPARENT_PROXY=$(gobool_to_int "${NGX_HTTP_TRANSPARENT_PROXY:-0}" 0)
+	export NGX_HTTP_TRANSPARENT_PROXY
+	if [ "${NGX_HTTP_TRANSPARENT_PROXY}" = 1 ] ; then
+		[ -n "${NGX_HTTP_FAKE_UA:-}" ] || NGX_HTTP_FAKE_UA=${_NGX_HTTP_FAKE_UA}
+		export NGX_HTTP_FAKE_UA
+
+		if [ -n "${NGX_HTTP_X_FORWARDED:-}" ] ; then
+			case "${NGX_HTTP_X_FORWARDED}" in
+			[Rr][Ee][Mm][Oo][Vv][Ee] ) ;;
+			* )
+				log_always "NGX_HTTP_X_FORWARDED: overridden to 'remove' due to NGX_HTTP_TRANSPARENT_PROXY=1"
+			;;
+			esac
+		fi
+		NGX_HTTP_X_FORWARDED=remove
+	fi
+
+	[ -n "${NGX_HTTP_X_FORWARDED:-}" ] || NGX_HTTP_X_FORWARDED=${_NGX_HTTP_X_FORWARDED}
+	case "${NGX_HTTP_X_FORWARDED}" in
+	[Pp][Aa][Ss][Ss] )
+		## adjust
+		NGX_HTTP_X_FORWARDED=pass
+	;;
+	[Rr][Ee][Mm][Oo][Vv][Ee] )
+		## adjust
+		NGX_HTTP_X_FORWARDED=remove
+	;;
+	* )
+		unset x
+		x=$(gobool_to_int "${NGX_HTTP_X_FORWARDED}")
+		case "$x" in
+		0 ) NGX_HTTP_X_FORWARDED=remove ;;
+		1 ) NGX_HTTP_X_FORWARDED=pass ;;
+		* )
+			log_always "NGX_HTTP_X_FORWARDED: unrecognized value: ${NGX_HTTP_X_FORWARDED}"
+			log_always "setting NGX_HTTP_X_FORWARDED=${_NGX_HTTP_X_FORWARDED}"
+			NGX_HTTP_X_FORWARDED=${_NGX_HTTP_X_FORWARDED}
+		;;
+		esac
+		unset x
+	;;
+	esac
+	export NGX_HTTP_X_FORWARDED
+
+	unset _NGX_HTTP_FAKE_UA _NGX_HTTP_X_FORWARDED
+fi

image-entry.d/72-merge-tree.sh

@@ -21,7 +21,7 @@ remap_path() {
 	esac
 }
 
-for n in ${NGX_DIRS_MERGE} ; do
+for n in j2cfg ${NGX_DIRS_MERGE} ; do
 	[ -n "$n" ] || continue
 
 	merged_dir="${merged_root}/$n"

image-entry.d/73-expand-templates.sh

@@ -37,12 +37,26 @@ for n in ${NGX_DIRS_MERGE} ; do
 	merge_dirs="${merge_dirs} $n/"
 done
 
-expand_dir_envsubst ${merge_dirs} || expand_error
-
 set -a
 J2CFG_PATH="${merged_root}/j2cfg"
 J2CFG_SEARCH_PATH="${merged_root}"
-set -a
+set +a
+
+## expand j2cfg templates first
+
+expand_dir_envsubst j2cfg/ || expand_error
+expand_dir_j2cfg j2cfg/    || expand_error
+
+## expand other templates
+
+expand_dir_envsubst ${merge_dirs} || expand_error
+
+unset j2cfg_dump
+j2cfg_dump="${volume_root}/diag.j2cfg.yml"
+
+j2cfg-dump > "${j2cfg_dump}" || expand_error
+
+export J2CFG_CONFIG="${j2cfg_dump}"
 
 expand_dir_j2cfg ${merge_dirs} || expand_error
 

image-entry.d/74-combine-tree.sh

@@ -88,7 +88,7 @@ while read -r old_path ; do
 done <<-EOF
 $(
 	set +e
-	for n in ${NGX_DIRS_MERGE} ; do
+	for n in j2cfg ${NGX_DIRS_MERGE} ; do
 		[ -n "$n" ] || continue
 
 		[ -d "${merged_root}/$n" ] || continue

image-entry.d/75-remove-merged-tree.sh

@@ -3,6 +3,8 @@ set -f
 
 . /image-entry.d/00-common.envsh
 
+IEP_RETAIN_MERGED_TREE=$(gobool_to_int "${IEP_RETAIN_MERGED_TREE:-0}" 0)
+
 if [ "${IEP_RETAIN_MERGED_TREE}" = 1 ] ; then
 	log_always "NOT removing merged tree: ${merged_root}/"
 else

image-entry.d/90-angie-config-test.sh

@@ -6,18 +6,20 @@ set -f
 ## Angie: unset core variable
 unset ANGIE ANGIE_BPF_MAPS
 
+_angie() {
+	angie -e stderr -g 'error_log /dev/stderr warn;' "$@"
+}
+
 ## merely debug test
 log_always 'test Angie configuration:'
 log_always '========================='
-angie -t
+_angie -t
 r=$?
 log_always '========================='
 
-## cleanup after test
-rm -f "${volume_root}/angie.pid"
-
 if [ $r = 0 ] ; then
 	log_always 'ready to run Angie'
+	_angie -T 2>&1 | cat > "${volume_root}/diag.angie.conf"
 else
 	log_always 'configuration test has failed, see above'
 	t=15
@@ -25,4 +27,7 @@ else
 	sleep $t
 fi
 
+## cleanup after test
+rm -f "${volume_root}/angie.pid"
+
 exit 0

image-entry.d/99-cleanup-env.envsh

@@ -3,6 +3,8 @@
 ## Angie: unset core variable
 unset ANGIE ANGIE_BPF_MAPS
 
+IEP_RETAIN_ENV=$(gobool_to_int "${IEP_RETAIN_ENV:-0}" 0)
+
 if [ "${IEP_RETAIN_ENV}" = 1 ] ; then
 	log_always "NOT removing following variables:"
 	sed -E '/^./s,^,  ,' >&2

image-entry.sh

@@ -93,10 +93,6 @@ IEP_INIT=$(gobool_to_int "${IEP_INIT:-0}" 0)
 # unexport IEP_INIT
 unset x ; x="${IEP_INIT}" ; unset IEP_INIT ; IEP_INIT="$x" ; unset x
 
-IEP_RETAIN_MERGED_TREE=$(gobool_to_int "${IEP_RETAIN_MERGED_TREE:-0}" 0)
-IEP_RETAIN_ENV=$(gobool_to_int "${IEP_RETAIN_ENV:-0}" 0)
-export IEP_RETAIN_MERGED_TREE IEP_RETAIN_ENV
-
 # IEP_TRACE=$(gobool_to_int "${IEP_TRACE:-0}" 0)
 IEP_DEBUG=$(gobool_to_int "${IEP_DEBUG:-0}" 0)
 IEP_VERBOSE=$(gobool_to_int "${IEP_VERBOSE:-${IEP_DEBUG}}" "${IEP_DEBUG}")

j2cfg/j2cfg-dump.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import os.path
+import sys
+
+
+def main():
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    import j2cfg
+
+    j = j2cfg.J2cfg(dump_only=True)
+    print(j.dump_config())
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

j2cfg/j2cfg/__init__.py

@@ -16,14 +16,16 @@ J2CFG_CONFIG_EXT = ['yml', 'yaml', 'json']
 
 
 class J2cfg:
-    def __init__(self, strict=True, config=None, config_path=None,
+    def __init__(self, strict=True, config_file=None, config_path=None,
-                 modules=None, search_path=None, template_suffix=None):
+                 modules=None, search_path=None, template_suffix=None,
+                 dump_only=False):
 
-        self.strict = strict
+        if dump_only is None:
-        if not isinstance(self.strict, bool):
+            self.dump_only = False
-            self.strict = True
+        else:
+            self.dump_only = bool(dump_only)
 
-        self.config_file = config or os.getenv('J2CFG_CONFIG')
+        self.config_file = config_file or os.getenv('J2CFG_CONFIG')
         if self.config_file is not None:
             self.config_file = str(self.config_file)
 
@@ -38,6 +40,87 @@ class J2cfg:
             self.config_path = any_to_str_list(self.config_path)
         self.config_path = uniq_str_list(self.config_path)
 
+        self.kwargs = {'j2cfg': {}}
+
+        def merge_dict_from_file(filename):
+            if filename is None:
+                return False
+            f = str(filename)
+            if f == '':
+                return False
+            if not os.path.exists(f):
+                return False
+            if not os.path.isfile(f):
+                print(
+                    f'J2cfg: not a file, skipping: {filename}',
+                    file=sys.stderr)
+                return False
+
+            if f.endswith('.yml') or f.endswith('.yaml'):
+                with open(f, mode='r', encoding='utf-8') as fx:
+                    for x in yaml.safe_load_all(fx):
+                        if not x:
+                            continue
+                        self.kwargs['j2cfg'] = merge_dict_recurse(
+                            self.kwargs['j2cfg'], x
+                        )
+                return True
+
+            if f.endswith('.json'):
+                with open(f, mode='r', encoding='utf-8') as fx:
+                    self.kwargs['j2cfg'] = merge_dict_recurse(
+                        self.kwargs['j2cfg'], json.load(fx)
+                    )
+                return True
+
+            print(
+                f'J2cfg: non-recognized name extension: {filename}',
+                file=sys.stderr)
+            return False
+
+        def merge_dict_default():
+            search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT])
+            search_flags = wcmatch.wcmatch.SYMLINKS
+
+            for d in self.config_path:
+                if not os.path.isdir(d):
+                    continue
+                m = wcmatch.wcmatch.WcMatch(d, search_pattern,
+                                            flags=search_flags)
+                for f in sorted(m.match()):
+                    if self.dump_only:
+                        real_f = os.path.realpath(f)
+                        if f == real_f:
+                            print(
+                                f'J2cfg: try loading {f}',
+                                file=sys.stderr
+                            )
+                        else:
+                            print(
+                                f'J2cfg: try loading {f} <- {real_f}',
+                                file=sys.stderr
+                            )
+                    merge_dict_from_file(f)
+
+        if self.config_file is None:
+            merge_dict_default()
+        else:
+            if os.path.isfile(self.config_file):
+                merge_dict_from_file(self.config_file)
+            else:
+                print(
+                    'J2cfg: J2cfg config file does not exist, skipping: '
+                    + f'{self.config_file}',
+                    file=sys.stderr
+                )
+
+        if self.dump_only:
+            return
+
+        self.strict = strict
+        if not isinstance(self.strict, bool):
+            self.strict = True
+
         self.search_path = search_path
         if self.search_path is None:
             self.search_path = os.getenv('J2CFG_SEARCH_PATH')
@@ -65,20 +148,19 @@ class J2cfg:
 
         self.template_suffix = template_suffix or os.getenv('J2CFG_SUFFIX')
         if self.template_suffix is None:
-            self.template_suffix = J2CFG_TEMPLATE_EXT
+            self.template_suffix = '' + J2CFG_TEMPLATE_EXT
         else:
             self.template_suffix = str(self.template_suffix)
             if self.template_suffix == '':
-                self.template_suffix = J2CFG_TEMPLATE_EXT
+                self.template_suffix = '' + J2CFG_TEMPLATE_EXT
         if not self.template_suffix.startswith('.'):
             self.template_suffix = '.' + self.template_suffix
 
-        self.kwargs = {
+        self.kwargs.update({
             'env': os.environ,
             'env_preserve': J2CFG_PRESERVE_ENVS.copy(),
             'env_passthrough': J2CFG_PASSTHROUGH_ENVS.copy(),
-            'j2cfg': {}
+        })
-        }
         for m in self.modules:
             if m in self.kwargs:
                 print(f'J2cfg: kwargs already has {m} key',
@@ -86,61 +168,6 @@ class J2cfg:
                 continue
             self.kwargs[m] = importlib.import_module(m)
 
-        def merge_dict_from_file(filename):
-            if filename is None:
-                return False
-            f = str(filename)
-            if f == '':
-                return False
-            if not os.path.exists(f):
-                return False
-            if not os.path.isfile(f):
-                print(
-                    f'J2cfg: not a file, skipping: {filename}',
-                    file=sys.stderr)
-                return False
-
-            if f.endswith('.yml') or f.endswith('.yaml'):
-                with open(f, mode='r', encoding='utf-8') as fx:
-                    x = yaml.safe_load(fx)
-                    self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x
-                return True
-
-            if f.endswith('.json'):
-                with open(f, mode='r', encoding='utf-8') as fx:
-                    x = json.load(fx)
-                    self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x
-                return True
-
-            print(
-                f'J2cfg: non-recognized name extension: {filename}',
-                file=sys.stderr)
-            return False
-
-        def merge_dict_default():
-            search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT])
-            search_flags = wcmatch.wcmatch.SYMLINKS
-
-            for d in self.config_path:
-                if not os.path.isdir(d):
-                    continue
-                m = wcmatch.wcmatch.WcMatch(d, search_pattern,
-                                            flags=search_flags)
-                for f in sorted(m.match()):
-                    merge_dict_from_file(f)
-
-        if self.config_file is None:
-            merge_dict_default()
-        else:
-            if os.path.isfile(self.config_file):
-                merge_dict_from_file(self.config_file)
-            else:
-                print(
-                    'J2cfg: J2cfg config file does not exist, skipping: '
-                    + f'{self.config_file}',
-                    file=sys.stderr
-                )
-
         self.j2fs_loaders = {
             d: jinja2.FileSystemLoader(
                 d, encoding='utf-8', followlinks=True,
@@ -164,7 +191,13 @@ class J2cfg:
 
         init_env(self.j2env)
 
+    def dump_config(self):
+        return yaml.safe_dump(self.kwargs['j2cfg'])
+
     def ensure_fs_loader_for(self, directory: str):
+        if self.dump_only:
+            raise ValueError('dump_only is True')
+
         if directory in self.j2fs_loaders:
             return
         self.j2fs_loaders[directory] = jinja2.FileSystemLoader(
@@ -172,6 +205,8 @@ class J2cfg:
         )
 
     def render_file(self, file_in, file_out=None) -> bool:
+        if self.dump_only:
+            raise ValueError('dump_only is True')
 
         def render_error(msg) -> bool:
             if self.strict:

j2cfg/j2cfg/functions.py

@@ -2,6 +2,7 @@ import collections.abc
 import itertools
 import pathlib
 import re
+import sys
 
 import jinja2
 
@@ -239,6 +240,41 @@ def sh_like_file_to_list(j2env, file_in: str) -> list:
     ))
 
 
+def merge_dict_recurse(d1, d2: dict) -> dict:
+    x = {} | d1
+
+    keys1 = set(x.keys())
+    keys2 = set(d2.keys())
+    common = keys1 & keys2
+    missing = keys2 - common
+
+    map1 = {k for k in common if is_mapping(x.get(k))}
+    seq1 = {k for k in common if is_sequence(x.get(k))}
+    misc1 = common - seq1 - map1
+
+    merge_safe = missing | misc1
+    x.update({k: d2.get(k) for k in merge_safe})
+
+    map_common = {k for k in map1 if is_mapping(d2.get(k))}
+    for k in map_common:
+        x[k] = merge_dict_recurse(x.get(k), d2.get(k))
+
+    seq_common = {k for k in seq1 if is_sequence(d2.get(k))}
+    for k in seq_common:
+        x[k] = uniq(list(x.get(k)) + list(d2.get(k)))
+
+    unmerged = (map1 - map_common) | (seq1 - seq_common)
+    for k in unmerged:
+        t1 = type(x.get(k))
+        t2 = type(d2.get(k))
+        print(
+            f'merge_dict_recurse(): skipping key {k}'
+            + f' due to type mismatch: {t1} vs. {t2}',
+            file=sys.stderr)
+
+    return x
+
+
 J2CFG_FILTERS = [
     any_to_env_dict,
     any_to_str_list,

scripts/angie-builtin-modules.sh

@@ -2,17 +2,17 @@
 set -f
 
 conf_dir='/etc/angie'
-conf_file="${conf_dir}/.none.conf"
-pid_file='/run/angie/none.pid'
 
-angie -q -e /dev/stderr -g "error_log /dev/stderr warn; pid ${pid_file};" -c "${conf_file}" -t
+## Angie: unset core variable
-r=$?
+unset ANGIE ANGIE_BPF_MAPS
-rm -f "${pid_file}"
+
-[ $r -eq 0 ] || exit $r
+_angie() {
+	angie -e stderr -g 'error_log /dev/stderr warn;' "$@"
+}
 
 t=$(mktemp) || exit $?
 
-angie -c "${conf_file}" -m 2>&1 | tee "$t" >/dev/null
+_angie -m 2>&1 | tee "$t" >/dev/null
 
 sed -En '/^ngx_(http|mail|stream)/d;/^ngx_(.+)_module$/{s//\1/;s/_filter$//;s/_/-/g;p}' < "$t" \
 | sort -uV > "${conf_dir}/builtin.core"

scripts/j2cfg-dump

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@"