From 6c59107425e1b8034b9b7d0519a1cce410a5db3b Mon Sep 17 00:00:00 2001 From: Konstantin Demin Date: Wed, 31 Jul 2024 14:04:15 +0300 Subject: [PATCH] refactoring --- Dockerfile | 1 + angie/.none.conf | 3 - angie/autoconf.dist/core-worker-env.conf.j2 | 2 +- angie/autoconf.dist/http-alt-svc.conf.j2 | 12 ++ .../http-request-headers-basic.conf.j2 | 26 +++ .../http-request-headers-forwarded.conf | 27 +++ angie/conf.dist/fastcgi/cache-bypass.conf.j2 | 15 ++ angie/conf.dist/fastcgi/headers.conf.j2 | 14 +- angie/conf.dist/grpc/headers.conf.j2 | 14 +- angie/conf.dist/http-grpc.conf | 3 - angie/conf.dist/http-proxy.conf | 1 + angie/conf.dist/http-quic-gso.conf | 3 + angie/conf.dist/http-response-headers.conf.j2 | 6 +- angie/conf.dist/http-v2.conf | 2 +- angie/conf.dist/http-v3.conf | 2 + angie/conf.dist/http2/param.conf | 2 + angie/conf.dist/http3/param.conf | 7 + angie/conf.dist/proxy/buffers.conf | 4 + angie/conf.dist/proxy/cache-bypass.conf.j2 | 15 ++ angie/conf.dist/proxy/headers.conf.j2 | 14 +- angie/conf.dist/proxy/version.conf | 1 + angie/conf.dist/scgi/cache-bypass.conf.j2 | 15 ++ angie/conf.dist/scgi/headers.conf.j2 | 14 +- angie/conf.dist/uwsgi/cache-bypass.conf.j2 | 15 ++ angie/conf.dist/uwsgi/headers.conf.j2 | 14 +- angie/j2cfg.dist/add-response-headers.yml | 11 -- angie/j2cfg.dist/cache-bypass.yml | 4 + angie/j2cfg.dist/core-worker-env.txt.j2 | 4 +- .../headers-request-transparent.yml.j2 | 12 ++ .../headers-request-x-forwarded.yml.j2 | 13 ++ angie/j2cfg.dist/headers-request.yml | 11 ++ angie/j2cfg.dist/headers-response.yml | 7 + angie/j2cfg.dist/remove-request-headers.yml | 3 - angie/j2cfg.dist/remove-response-headers.yml | 12 -- angie/modsecurity.dist/rules.conf | 2 +- angie/snip.dist/disable-compression.j2 | 8 +- image-entry.d/01-defaults.envsh | 2 +- image-entry.d/21-http-modules.envsh | 18 ++ image-entry.d/23-http-forward-headers.envsh | 56 ++++++ image-entry.d/72-merge-tree.sh | 2 +- image-entry.d/73-expand-templates.sh | 20 ++- image-entry.d/74-combine-tree.sh | 2 +- image-entry.d/75-remove-merged-tree.sh | 2 + image-entry.d/90-angie-config-test.sh | 13 +- image-entry.d/99-cleanup-env.envsh | 2 + image-entry.sh | 4 - j2cfg/j2cfg-dump.py | 18 ++ j2cfg/j2cfg/__init__.py | 167 +++++++++++------- j2cfg/j2cfg/functions.py | 36 ++++ scripts/angie-builtin-modules.sh | 14 +- scripts/j2cfg-dump | 2 + 51 files changed, 511 insertions(+), 166 deletions(-) delete mode 100644 angie/.none.conf create mode 100644 angie/autoconf.dist/http-alt-svc.conf.j2 create mode 100644 angie/autoconf.dist/http-request-headers-basic.conf.j2 create mode 100644 angie/autoconf.dist/http-request-headers-forwarded.conf create mode 100644 angie/conf.dist/fastcgi/cache-bypass.conf.j2 create mode 100644 angie/conf.dist/http-proxy.conf create mode 100644 angie/conf.dist/http-quic-gso.conf create mode 100644 angie/conf.dist/http-v3.conf create mode 100644 angie/conf.dist/http2/param.conf create mode 100644 angie/conf.dist/http3/param.conf create mode 100644 angie/conf.dist/proxy/buffers.conf create mode 100644 angie/conf.dist/proxy/cache-bypass.conf.j2 create mode 100644 angie/conf.dist/proxy/version.conf create mode 100644 angie/conf.dist/scgi/cache-bypass.conf.j2 create mode 100644 angie/conf.dist/uwsgi/cache-bypass.conf.j2 delete mode 100644 angie/j2cfg.dist/add-response-headers.yml create mode 100644 angie/j2cfg.dist/cache-bypass.yml create mode 100644 angie/j2cfg.dist/headers-request-transparent.yml.j2 create mode 100644 angie/j2cfg.dist/headers-request-x-forwarded.yml.j2 create mode 100644 angie/j2cfg.dist/headers-request.yml create mode 100644 angie/j2cfg.dist/headers-response.yml delete mode 100644 angie/j2cfg.dist/remove-request-headers.yml delete mode 100644 angie/j2cfg.dist/remove-response-headers.yml create mode 100755 image-entry.d/23-http-forward-headers.envsh create mode 100755 j2cfg/j2cfg-dump.py create mode 100755 scripts/j2cfg-dump diff --git a/Dockerfile b/Dockerfile index 10de92d..8077d1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -174,6 +174,7 @@ RUN x='angie-builtin-modules.sh' ; \ ## misc tools RUN apt-install.sh \ brotli \ + curl \ zstd \ ; \ apt-clean.sh diff --git a/angie/.none.conf b/angie/.none.conf deleted file mode 100644 index c337864..0000000 --- a/angie/.none.conf +++ /dev/null @@ -1,3 +0,0 @@ -daemon off; -master_process off; -events {} diff --git a/angie/autoconf.dist/core-worker-env.conf.j2 b/angie/autoconf.dist/core-worker-env.conf.j2 index bb8f0e7..61a6ce1 100644 --- a/angie/autoconf.dist/core-worker-env.conf.j2 +++ b/angie/autoconf.dist/core-worker-env.conf.j2 @@ -28,4 +28,4 @@ env {{ k }}; {#- {%- set v = c_env[k] %} #} ## env {{ k }}={{ c_env[k].__repr__() }}; {%- endfor %} -{%- endif %} +{%- endif %} \ No newline at end of file diff --git a/angie/autoconf.dist/http-alt-svc.conf.j2 b/angie/autoconf.dist/http-alt-svc.conf.j2 new file mode 100644 index 0000000..f764b23 --- /dev/null +++ b/angie/autoconf.dist/http-alt-svc.conf.j2 @@ -0,0 +1,12 @@ +{#- prologue -#} +{%- set extra_proto = ['v3', 'v2'] -%} +{%- set confload = ( env.NGX_HTTP_CONFLOAD or '' ) | str_split_to_list -%} +{%- set proto = confload | list_intersect(extra_proto) -%} +{#- ALPN mapping -#} +{%- set proto = proto | re_sub('^v2$', 'h2=":443"; ma=3600') -%} +{%- set proto = proto | re_sub('^v3$', 'h3=":443"; ma=3600') -%} +{#- main part -#} +{%- if proto %} +{#- TODO: precise quotation #} +add_header Alt-Svc {{ (proto | join(', ')).__repr__() }}; +{%- endif %} \ No newline at end of file diff --git a/angie/autoconf.dist/http-request-headers-basic.conf.j2 b/angie/autoconf.dist/http-request-headers-basic.conf.j2 new file mode 100644 index 0000000..65a5166 --- /dev/null +++ b/angie/autoconf.dist/http-request-headers-basic.conf.j2 @@ -0,0 +1,26 @@ +map $http_upgrade + $req_connection +{ + default upgrade; + "" ""; +} + +map $http_user_agent + $req_user_agent +{ + default $http_user_agent; +{%- if env.NGX_HTTP_FAKE_UA %} + ## merely fake + "" {{ env.NGX_HTTP_FAKE_UA.__repr__() }}; +{%- else %} + "" "Angie/$angie_version"; +{%- endif %} +} + +map $http_accept + $req_accept +{ + volatile; + default $http_accept; + "" "*/*"; +} \ No newline at end of file diff --git a/angie/autoconf.dist/http-request-headers-forwarded.conf b/angie/autoconf.dist/http-request-headers-forwarded.conf new file mode 100644 index 0000000..afb55c6 --- /dev/null +++ b/angie/autoconf.dist/http-request-headers-forwarded.conf @@ -0,0 +1,27 @@ +## ref: +## - https://www.digitalocean.com/community/tools/nginx?domains.0.reverseProxy.reverseProxy=true +map $remote_addr + $proxy_forwarded_elem +{ + ## IPv4 addresses can be sent as-is + ~^[0-9.]+$ "for=$remote_addr"; + ## IPv6 addresses need to be bracketed and quoted + ~^[0-9A-Fa-f:.]+$ "for=\"[$remote_addr]\""; + ## Unix domain socket names cannot be represented in RFC 7239 syntax + default "for=unknown"; +} + +## ref: +## - https://www.digitalocean.com/community/tools/nginx?domains.0.reverseProxy.reverseProxy=true +## - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded +map $http_forwarded + $proxy_add_forwarded +{ + volatile; + + ## if the incoming Forwarded header is syntactically valid, append to it + "~^(,[ \\t]*)*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*([ \\t]*,([ \\t]*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*)?)*$" "$http_forwarded, $proxy_forwarded_elem"; + + ## otherwise, replace it + default "$proxy_forwarded_elem"; +} \ No newline at end of file diff --git a/angie/conf.dist/fastcgi/cache-bypass.conf.j2 b/angie/conf.dist/fastcgi/cache-bypass.conf.j2 new file mode 100644 index 0000000..18d6fe9 --- /dev/null +++ b/angie/conf.dist/fastcgi/cache-bypass.conf.j2 @@ -0,0 +1,15 @@ +{#- TODO: precise quotation -#} +{%- set cache_bypass = j2cfg.fastcgi_cache_bypass or j2cfg.cache_bypass or [] -%} +{%- if cache_bypass -%} +## disable (response) cache under following conditions +fastcgi_cache_bypass +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +fastcgi_no_cache +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +{%- endif -%} \ No newline at end of file diff --git a/angie/conf.dist/fastcgi/headers.conf.j2 b/angie/conf.dist/fastcgi/headers.conf.j2 index ecec9d5..fbbb649 100644 --- a/angie/conf.dist/fastcgi/headers.conf.j2 +++ b/angie/conf.dist/fastcgi/headers.conf.j2 @@ -1,13 +1,13 @@ ## hide/remove request headers -{%- set req_hdr_list = j2cfg.fastcgi_remove_request_headers or j2cfg.remove_request_headers or [] -%} -{%- set req_hdr_list = req_hdr_list | any_to_str_list | as_cgi_header -%} -{%- for h in req_hdr_list %} -fastcgi_param {{ h }} ""; +{%- set req_hdr_dict = j2cfg.fastcgi_request_headers or j2cfg.request_headers or {} -%} +{%- for h, v in req_hdr_dict.items() %} +{#- TODO: precise quotation #} +fastcgi_param {{ h | as_cgi_header }} {{ v.__repr__() }}; {%- endfor %} ## hide response headers -{%- set resp_hdr_list = j2cfg.fastcgi_remove_response_headers or j2cfg.remove_response_headers or [] -%} -{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%} +{%- set resp_hdr_dict = j2cfg.fastcgi_response_headers or j2cfg.response_headers or {} -%} +{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%} {%- for h in resp_hdr_list %} fastcgi_hide_header {{ h }}; -{%- endfor %} +{%- endfor %} \ No newline at end of file diff --git a/angie/conf.dist/grpc/headers.conf.j2 b/angie/conf.dist/grpc/headers.conf.j2 index ab7d433..b2d4ae4 100644 --- a/angie/conf.dist/grpc/headers.conf.j2 +++ b/angie/conf.dist/grpc/headers.conf.j2 @@ -1,13 +1,13 @@ ## hide/remove request headers -{%- set req_hdr_list = j2cfg.grpc_remove_request_headers or j2cfg.remove_request_headers or [] -%} -{%- set req_hdr_list = req_hdr_list | any_to_str_list | uniq_str_list -%} -{%- for h in req_hdr_list %} -grpc_set_header {{ h }} ""; +{%- set req_hdr_dict = j2cfg.grpc_request_headers or j2cfg.request_headers or {} -%} +{%- for h, v in req_hdr_dict.items() %} +{#- TODO: precise quotation #} +grpc_set_header {{ h }} {{ v.__repr__() }}; {%- endfor %} ## hide response headers -{%- set resp_hdr_list = j2cfg.grpc_remove_response_headers or j2cfg.remove_response_headers or [] -%} -{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%} +{%- set resp_hdr_dict = j2cfg.grpc_response_headers or j2cfg.response_headers or {} -%} +{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%} {%- for h in resp_hdr_list %} grpc_hide_header {{ h }}; -{%- endfor %} +{%- endfor %} \ No newline at end of file diff --git a/angie/conf.dist/http-grpc.conf b/angie/conf.dist/http-grpc.conf index d6c368f..cd17a8d 100644 --- a/angie/conf.dist/http-grpc.conf +++ b/angie/conf.dist/http-grpc.conf @@ -1,4 +1 @@ -## this should be enabled explicitly to avoid config mess -# include conf.d/http-v2.conf; - include conf.d/grpc/*.conf; \ No newline at end of file diff --git a/angie/conf.dist/http-proxy.conf b/angie/conf.dist/http-proxy.conf new file mode 100644 index 0000000..719ddd7 --- /dev/null +++ b/angie/conf.dist/http-proxy.conf @@ -0,0 +1 @@ +include conf.d/proxy/*.conf; \ No newline at end of file diff --git a/angie/conf.dist/http-quic-gso.conf b/angie/conf.dist/http-quic-gso.conf new file mode 100644 index 0000000..2b55f0e --- /dev/null +++ b/angie/conf.dist/http-quic-gso.conf @@ -0,0 +1,3 @@ +quic_gso on; + +proxy_quic_gso on; \ No newline at end of file diff --git a/angie/conf.dist/http-response-headers.conf.j2 b/angie/conf.dist/http-response-headers.conf.j2 index 645a02c..142b081 100644 --- a/angie/conf.dist/http-response-headers.conf.j2 +++ b/angie/conf.dist/http-response-headers.conf.j2 @@ -1,6 +1,6 @@ ## add response headers -{%- set resp_hdr_list = ( j2cfg.add_response_headers or {} ) -%} -{%- for h, v in resp_hdr_list.items() %} +{%- set resp_hdr_dict = j2cfg.response_headers or {} -%} +{%- for h, v in resp_hdr_dict.items() %} {#- TODO: precise quotation #} add_header {{ h }} {{ v.__repr__() }}; -{%- endfor %} +{%- endfor %} \ No newline at end of file diff --git a/angie/conf.dist/http-v2.conf b/angie/conf.dist/http-v2.conf index cfba10a..2a8de50 100644 --- a/angie/conf.dist/http-v2.conf +++ b/angie/conf.dist/http-v2.conf @@ -1,2 +1,2 @@ -http2_chunk_size 16k; +include conf.d/http2/*.conf; http2 on; \ No newline at end of file diff --git a/angie/conf.dist/http-v3.conf b/angie/conf.dist/http-v3.conf new file mode 100644 index 0000000..23a6672 --- /dev/null +++ b/angie/conf.dist/http-v3.conf @@ -0,0 +1,2 @@ +include conf.d/http3/*.conf; +http3 on; \ No newline at end of file diff --git a/angie/conf.dist/http2/param.conf b/angie/conf.dist/http2/param.conf new file mode 100644 index 0000000..2645e78 --- /dev/null +++ b/angie/conf.dist/http2/param.conf @@ -0,0 +1,2 @@ +http2_chunk_size 16k; +http2_body_preread_size 64k; \ No newline at end of file diff --git a/angie/conf.dist/http3/param.conf b/angie/conf.dist/http3/param.conf new file mode 100644 index 0000000..a20caca --- /dev/null +++ b/angie/conf.dist/http3/param.conf @@ -0,0 +1,7 @@ +http3_max_concurrent_streams 128; #default +http3_stream_buffer_size 64k; #default +quic_active_connection_id_limit 3; + +proxy_http3_max_concurrent_streams 128; #default +proxy_http3_stream_buffer_size 64k; #default +proxy_quic_active_connection_id_limit 3; \ No newline at end of file diff --git a/angie/conf.dist/proxy/buffers.conf b/angie/conf.dist/proxy/buffers.conf new file mode 100644 index 0000000..7e69216 --- /dev/null +++ b/angie/conf.dist/proxy/buffers.conf @@ -0,0 +1,4 @@ +proxy_buffers 16 16k; +proxy_buffer_size 16k; +proxy_busy_buffers_size 32k; +proxy_temp_file_write_size 32k; \ No newline at end of file diff --git a/angie/conf.dist/proxy/cache-bypass.conf.j2 b/angie/conf.dist/proxy/cache-bypass.conf.j2 new file mode 100644 index 0000000..fe6a29c --- /dev/null +++ b/angie/conf.dist/proxy/cache-bypass.conf.j2 @@ -0,0 +1,15 @@ +{#- TODO: precise quotation -#} +{%- set cache_bypass = j2cfg.proxy_cache_bypass or j2cfg.cache_bypass or [] -%} +{%- if cache_bypass -%} +## disable (response) cache under following conditions +proxy_cache_bypass +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +proxy_no_cache +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +{%- endif -%} \ No newline at end of file diff --git a/angie/conf.dist/proxy/headers.conf.j2 b/angie/conf.dist/proxy/headers.conf.j2 index c2d1b7e..76376f4 100644 --- a/angie/conf.dist/proxy/headers.conf.j2 +++ b/angie/conf.dist/proxy/headers.conf.j2 @@ -1,13 +1,13 @@ ## hide/remove request headers -{%- set req_hdr_list = j2cfg.proxy_remove_request_headers or j2cfg.remove_request_headers or [] -%} -{%- set req_hdr_list = req_hdr_list | any_to_str_list | uniq_str_list -%} -{%- for h in req_hdr_list %} -proxy_set_header {{ h }} ""; +{%- set req_hdr_dict = j2cfg.proxy_request_headers or j2cfg.request_headers or {} -%} +{%- for h, v in req_hdr_dict.items() %} +{#- TODO: precise quotation #} +proxy_set_header {{ h }} {{ v.__repr__() }}; {%- endfor %} ## hide response headers -{%- set resp_hdr_list = j2cfg.proxy_remove_response_headers or j2cfg.remove_response_headers or [] -%} -{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%} +{%- set resp_hdr_dict = j2cfg.proxy_response_headers or j2cfg.response_headers or {} -%} +{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%} {%- for h in resp_hdr_list %} proxy_hide_header {{ h }}; -{%- endfor %} +{%- endfor %} \ No newline at end of file diff --git a/angie/conf.dist/proxy/version.conf b/angie/conf.dist/proxy/version.conf new file mode 100644 index 0000000..8d0948b --- /dev/null +++ b/angie/conf.dist/proxy/version.conf @@ -0,0 +1 @@ +proxy_http_version 1.1; \ No newline at end of file diff --git a/angie/conf.dist/scgi/cache-bypass.conf.j2 b/angie/conf.dist/scgi/cache-bypass.conf.j2 new file mode 100644 index 0000000..1e1c44a --- /dev/null +++ b/angie/conf.dist/scgi/cache-bypass.conf.j2 @@ -0,0 +1,15 @@ +{#- TODO: precise quotation -#} +{%- set cache_bypass = j2cfg.scgi_cache_bypass or j2cfg.cache_bypass or [] -%} +{%- if cache_bypass -%} +## disable (response) cache under following conditions +scgi_cache_bypass +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +scgi_no_cache +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +{%- endif -%} \ No newline at end of file diff --git a/angie/conf.dist/scgi/headers.conf.j2 b/angie/conf.dist/scgi/headers.conf.j2 index 8535057..e021411 100644 --- a/angie/conf.dist/scgi/headers.conf.j2 +++ b/angie/conf.dist/scgi/headers.conf.j2 @@ -1,13 +1,13 @@ ## hide/remove request headers -{%- set req_hdr_list = j2cfg.scgi_remove_request_headers or j2cfg.remove_request_headers or [] -%} -{%- set req_hdr_list = req_hdr_list | any_to_str_list | as_cgi_header -%} -{%- for h in req_hdr_list %} -scgi_param {{ h }} ""; +{%- set req_hdr_dict = j2cfg.scgi_request_headers or j2cfg.request_headers or {} -%} +{%- for h, v in req_hdr_dict.items() %} +{#- TODO: precise quotation #} +scgi_param {{ h | as_cgi_header }} {{ v.__repr__() }}; {%- endfor %} ## hide response headers -{%- set resp_hdr_list = j2cfg.scgi_remove_response_headers or j2cfg.remove_response_headers or [] -%} -{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%} +{%- set resp_hdr_dict = j2cfg.scgi_response_headers or j2cfg.response_headers or {} -%} +{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%} {%- for h in resp_hdr_list %} scgi_hide_header {{ h }}; -{%- endfor %} +{%- endfor %} \ No newline at end of file diff --git a/angie/conf.dist/uwsgi/cache-bypass.conf.j2 b/angie/conf.dist/uwsgi/cache-bypass.conf.j2 new file mode 100644 index 0000000..2da45cb --- /dev/null +++ b/angie/conf.dist/uwsgi/cache-bypass.conf.j2 @@ -0,0 +1,15 @@ +{#- TODO: precise quotation -#} +{%- set cache_bypass = j2cfg.uwsgi_cache_bypass or j2cfg.cache_bypass or [] -%} +{%- if cache_bypass -%} +## disable (response) cache under following conditions +uwsgi_cache_bypass +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +uwsgi_no_cache +{%- for v in cache_bypass %} + {{ v.__repr__() }} +{%- endfor %} +; +{%- endif -%} \ No newline at end of file diff --git a/angie/conf.dist/uwsgi/headers.conf.j2 b/angie/conf.dist/uwsgi/headers.conf.j2 index e727e97..3226785 100644 --- a/angie/conf.dist/uwsgi/headers.conf.j2 +++ b/angie/conf.dist/uwsgi/headers.conf.j2 @@ -1,13 +1,13 @@ ## hide/remove request headers -{%- set req_hdr_list = j2cfg.uwsgi_remove_request_headers or j2cfg.remove_request_headers or [] -%} -{%- set req_hdr_list = req_hdr_list | any_to_str_list | as_cgi_header -%} -{%- for h in req_hdr_list %} -uwsgi_param {{ h }} ""; +{%- set req_hdr_dict = j2cfg.uwsgi_request_headers or j2cfg.request_headers or {} -%} +{%- for h, v in req_hdr_dict.items() %} +{#- TODO: precise quotation #} +uwsgi_param {{ h | as_cgi_header }} {{ v.__repr__() }}; {%- endfor %} ## hide response headers -{%- set resp_hdr_list = j2cfg.uwsgi_remove_response_headers or j2cfg.remove_response_headers or [] -%} -{%- set resp_hdr_list = resp_hdr_list | any_to_str_list | uniq_str_list -%} +{%- set resp_hdr_dict = j2cfg.uwsgi_response_headers or j2cfg.response_headers or {} -%} +{%- set resp_hdr_list = resp_hdr_dict | dict_keys -%} {%- for h in resp_hdr_list %} uwsgi_hide_header {{ h }}; -{%- endfor %} +{%- endfor %} \ No newline at end of file diff --git a/angie/j2cfg.dist/add-response-headers.yml b/angie/j2cfg.dist/add-response-headers.yml deleted file mode 100644 index 7ad205a..0000000 --- a/angie/j2cfg.dist/add-response-headers.yml +++ /dev/null @@ -1,11 +0,0 @@ -add_response_headers: - Access-Control-Allow-Origin: "*" - Access-Control-Allow-Headers: "Origin, X-Requested-With, Content-Type, Accept, Authorization" - Access-Control-Allow-Methods: "GET, HEAD, POST, PUT, DELETE, OPTIONS" - Content-Security-Policy: "default-src 'self' http: https: ws: wss: data: blob: 'unsafe-inline' 'unsafe-eval' ; frame-ancestors 'self';" - Permissions-Policy: "microphone=(), camera=(), geolocation=(), interest-cohort=()" - Referrer-Policy: "no-referrer-when-downgrade" - Strict-Transport-Security: "max-age=31536000; includeSubDomains; preload" - X-Content-Type-Options: "nosniff" - X-Frame-Options: "SAMEORIGIN" - X-XSS-Protection: "1; mode=block" diff --git a/angie/j2cfg.dist/cache-bypass.yml b/angie/j2cfg.dist/cache-bypass.yml new file mode 100644 index 0000000..128a366 --- /dev/null +++ b/angie/j2cfg.dist/cache-bypass.yml @@ -0,0 +1,4 @@ +cache_bypass: +- '$http_authorization' +- '$http_pragma' +- '$http_upgrade' \ No newline at end of file diff --git a/angie/j2cfg.dist/core-worker-env.txt.j2 b/angie/j2cfg.dist/core-worker-env.txt.j2 index 3564d09..293dce3 100644 --- a/angie/j2cfg.dist/core-worker-env.txt.j2 +++ b/angie/j2cfg.dist/core-worker-env.txt.j2 @@ -4,6 +4,6 @@ {%- set c_vars_passthrough = c_env | dict_empty_keys -%} {%- set vars_passthrough = ((env_passthrough | list_diff(c_vars)) + c_vars_passthrough) | uniq | list_intersect(env | dict_keys) -%} {#- main part -#} -{%- for k in vars_passthrough -%} +{%- for k in vars_passthrough %} {{ k }} -{% endfor -%} +{%- endfor %} \ No newline at end of file diff --git a/angie/j2cfg.dist/headers-request-transparent.yml.j2 b/angie/j2cfg.dist/headers-request-transparent.yml.j2 new file mode 100644 index 0000000..9f287db --- /dev/null +++ b/angie/j2cfg.dist/headers-request-transparent.yml.j2 @@ -0,0 +1,12 @@ +{% if env.NGX_HTTP_TRANSPARENT_PROXY == '0' %} +request_headers: + Host: '$proxy_host' + X-Real-IP: '$remote_addr' + ## '$proxy_add_forwarded' is defined in /angie/autoconf.dist/http-request-headers-forwarded.conf + Forwarded: '$proxy_add_forwarded' +{% elif env.NGX_HTTP_TRANSPARENT_PROXY == '1' %} +request_headers: + Host: '$host' + X-Real-IP: '' + Forwarded: '' +{% endif %} \ No newline at end of file diff --git a/angie/j2cfg.dist/headers-request-x-forwarded.yml.j2 b/angie/j2cfg.dist/headers-request-x-forwarded.yml.j2 new file mode 100644 index 0000000..79b88ff --- /dev/null +++ b/angie/j2cfg.dist/headers-request-x-forwarded.yml.j2 @@ -0,0 +1,13 @@ +{% if env.NGX_HTTP_X_FORWARDED == 'pass' %} +request_headers: + X-Forwarded-Proto: '$scheme' + X-Forwarded-Host: '$host' + X-Forwarded-Port: '$server_port' + X-Forwarded-For: '$proxy_add_x_forwarded_for' +{% elif env.NGX_HTTP_X_FORWARDED == 'remove' %} +request_headers: + X-Forwarded-Proto: '' + X-Forwarded-Host: '' + X-Forwarded-Port: '' + X-Forwarded-For: '' +{% endif %} \ No newline at end of file diff --git a/angie/j2cfg.dist/headers-request.yml b/angie/j2cfg.dist/headers-request.yml new file mode 100644 index 0000000..4cb6d65 --- /dev/null +++ b/angie/j2cfg.dist/headers-request.yml @@ -0,0 +1,11 @@ +request_headers: + ## do not pass Accept-Encoding to backend + Accept-Encoding: "" + ## '$req_accept' is defined in /angie/autoconf.dist/http-request-headers-basic.conf.j2 + Accept: '$req_accept' + ## '$req_connection' is defined in /angie/autoconf.dist/http-request-headers-basic.conf.j2 + Connection: '$req_connection' + Upgrade: '$http_upgrade' + Early-Data: '$ssl_early_data' + ## '$req_user_agent' is defined in /angie/autoconf.dist/http-request-headers-basic.conf.j2 + User-Agent: '$req_user_agent' \ No newline at end of file diff --git a/angie/j2cfg.dist/headers-response.yml b/angie/j2cfg.dist/headers-response.yml new file mode 100644 index 0000000..4a2466e --- /dev/null +++ b/angie/j2cfg.dist/headers-response.yml @@ -0,0 +1,7 @@ +response_headers: + Permissions-Policy: "microphone=(), camera=(), geolocation=(), interest-cohort=()" + Referrer-Policy: "no-referrer-when-downgrade" + Strict-Transport-Security: "max-age=15724800; includeSubDomains; preload" + X-Content-Type-Options: "nosniff" + X-Frame-Options: "SAMEORIGIN" + X-XSS-Protection: "1; mode=block" \ No newline at end of file diff --git a/angie/j2cfg.dist/remove-request-headers.yml b/angie/j2cfg.dist/remove-request-headers.yml deleted file mode 100644 index e2f8aef..0000000 --- a/angie/j2cfg.dist/remove-request-headers.yml +++ /dev/null @@ -1,3 +0,0 @@ -remove_request_headers: - ## do not pass Accept-Encoding to backend -- Accept-Encoding \ No newline at end of file diff --git a/angie/j2cfg.dist/remove-response-headers.yml b/angie/j2cfg.dist/remove-response-headers.yml deleted file mode 100644 index 9c43c15..0000000 --- a/angie/j2cfg.dist/remove-response-headers.yml +++ /dev/null @@ -1,12 +0,0 @@ -remove_response_headers: -- Access-Control-Allow-Headers -- Access-Control-Allow-Methods -- Access-Control-Allow-Origin -- Content-Security-Policy -- Permissions-Policy -- Referrer-Policy -- Strict-Transport-Security -- Vary -- X-Content-Type-Options -- X-Frame-Options -- X-XSS-Protection \ No newline at end of file diff --git a/angie/modsecurity.dist/rules.conf b/angie/modsecurity.dist/rules.conf index f6892b2..997e7df 100644 --- a/angie/modsecurity.dist/rules.conf +++ b/angie/modsecurity.dist/rules.conf @@ -10,7 +10,7 @@ Include modsecurity.conf # w=$(mktemp -d) ; : "${w:?}" # cd "$w/" # tarball="coreruleset.tar.gz" -# /usr/lib/apt/apt-helper download-file "${uri}" "${tarball}" +# curl -Lo "${tarball}" "${uri}" # mkdir coreruleset # tar -C ./coreruleset --strip-components=1 -xf "${tarball}" # rm -f "${tarball}" ; unset tarball diff --git a/angie/snip.dist/disable-compression.j2 b/angie/snip.dist/disable-compression.j2 index 599fdde..3df37eb 100644 --- a/angie/snip.dist/disable-compression.j2 +++ b/angie/snip.dist/disable-compression.j2 @@ -1,8 +1,8 @@ {#- safe to specify all the time -#} gzip off; +{%- set extra_comp_modules = ['brotli', 'zstd'] -%} {%- set modules = ( env.NGX_HTTP_MODULES or '' ) | str_split_to_list -%} -{%- for ext_comp in ['brotli', 'zstd'] %} -{%- if ext_comp in modules %} -{{ ext_comp }} off; -{%- endif %} +{%- set comp_modules = modules | list_intersect(extra_comp_modules) | sort -%} +{%- for comp in comp_modules %} +{{ comp }} off; {%- endfor %} \ No newline at end of file diff --git a/image-entry.d/01-defaults.envsh b/image-entry.d/01-defaults.envsh index f10e867..531308d 100755 --- a/image-entry.d/01-defaults.envsh +++ b/image-entry.d/01-defaults.envsh @@ -19,7 +19,7 @@ if [ "${NGX_HTTP}${NGX_MAIL}${NGX_STREAM}" = '000' ] ; then fi unset default_dirs_merge default_dirs_link -default_dirs_merge='autoconf conf j2cfg mod modules site snip' +default_dirs_merge='autoconf conf mod modules site snip' default_dirs_link='' if [ "${NGX_PROCESS_STATIC}" = 1 ] ; then diff --git a/image-entry.d/21-http-modules.envsh b/image-entry.d/21-http-modules.envsh index 3154fde..a9450dd 100755 --- a/image-entry.d/21-http-modules.envsh +++ b/image-entry.d/21-http-modules.envsh @@ -3,6 +3,11 @@ if [ "${NGX_HTTP}" = 0 ] ; then unset NGX_HTTP_NO_PROXY NGX_HTTP_WITH_MODSECURITY else + NGX_HTTP_NO_PROXY=$(gobool_to_int "${NGX_HTTP_NO_PROXY:-0}" 0) + if [ "${NGX_HTTP_NO_PROXY}" = 0 ] ; then + NGX_HTTP_CONFLOAD=$(append_list "${NGX_HTTP_CONFLOAD}" proxy) + fi + unset http_modules http_confload http_modules= http_confload="${NGX_HTTP_CONFLOAD:-}" @@ -56,6 +61,19 @@ else done unset i + ## grpc depends on http/2 + if list_have_item "${NGX_HTTP_CONFLOAD}" grpc ; then + unset want_http2 + want_http2=0 + if ! list_have_item "${NGX_HTTP_CONFLOAD}" v2 ; then + want_http2=1 + fi + if [ "${want_http2}" = 1 ] ; then + NGX_HTTP_CONFLOAD=$(append_list "${NGX_HTTP_CONFLOAD}" v2) + fi + unset want_http2 + fi + set -a NGX_HTTP_MODULES="${http_modules}" NGX_HTTP_CONFLOAD=$(sort_dedup_list "${http_confload}") diff --git a/image-entry.d/23-http-forward-headers.envsh b/image-entry.d/23-http-forward-headers.envsh new file mode 100755 index 0000000..c96dc32 --- /dev/null +++ b/image-entry.d/23-http-forward-headers.envsh @@ -0,0 +1,56 @@ +#!/bin/sh + +if [ "${NGX_HTTP}" = 0 ] ; then + unset NGX_HTTP_TRANSPARENT_PROXY NGX_HTTP_FAKE_UA NGX_HTTP_X_FORWARDED +else + unset _NGX_HTTP_FAKE_UA _NGX_HTTP_X_FORWARDED + ## here should be SANE defaults (!) + _NGX_HTTP_FAKE_UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36' + _NGX_HTTP_X_FORWARDED=pass + + NGX_HTTP_TRANSPARENT_PROXY=$(gobool_to_int "${NGX_HTTP_TRANSPARENT_PROXY:-0}" 0) + export NGX_HTTP_TRANSPARENT_PROXY + if [ "${NGX_HTTP_TRANSPARENT_PROXY}" = 1 ] ; then + [ -n "${NGX_HTTP_FAKE_UA:-}" ] || NGX_HTTP_FAKE_UA=${_NGX_HTTP_FAKE_UA} + export NGX_HTTP_FAKE_UA + + if [ -n "${NGX_HTTP_X_FORWARDED:-}" ] ; then + case "${NGX_HTTP_X_FORWARDED}" in + [Rr][Ee][Mm][Oo][Vv][Ee] ) ;; + * ) + log_always "NGX_HTTP_X_FORWARDED: overridden to 'remove' due to NGX_HTTP_TRANSPARENT_PROXY=1" + ;; + esac + fi + NGX_HTTP_X_FORWARDED=remove + fi + + [ -n "${NGX_HTTP_X_FORWARDED:-}" ] || NGX_HTTP_X_FORWARDED=${_NGX_HTTP_X_FORWARDED} + case "${NGX_HTTP_X_FORWARDED}" in + [Pp][Aa][Ss][Ss] ) + ## adjust + NGX_HTTP_X_FORWARDED=pass + ;; + [Rr][Ee][Mm][Oo][Vv][Ee] ) + ## adjust + NGX_HTTP_X_FORWARDED=remove + ;; + * ) + unset x + x=$(gobool_to_int "${NGX_HTTP_X_FORWARDED}") + case "$x" in + 0 ) NGX_HTTP_X_FORWARDED=remove ;; + 1 ) NGX_HTTP_X_FORWARDED=pass ;; + * ) + log_always "NGX_HTTP_X_FORWARDED: unrecognized value: ${NGX_HTTP_X_FORWARDED}" + log_always "setting NGX_HTTP_X_FORWARDED=${_NGX_HTTP_X_FORWARDED}" + NGX_HTTP_X_FORWARDED=${_NGX_HTTP_X_FORWARDED} + ;; + esac + unset x + ;; + esac + export NGX_HTTP_X_FORWARDED + + unset _NGX_HTTP_FAKE_UA _NGX_HTTP_X_FORWARDED +fi diff --git a/image-entry.d/72-merge-tree.sh b/image-entry.d/72-merge-tree.sh index e20f6a9..01cbc1a 100755 --- a/image-entry.d/72-merge-tree.sh +++ b/image-entry.d/72-merge-tree.sh @@ -21,7 +21,7 @@ remap_path() { esac } -for n in ${NGX_DIRS_MERGE} ; do +for n in j2cfg ${NGX_DIRS_MERGE} ; do [ -n "$n" ] || continue merged_dir="${merged_root}/$n" diff --git a/image-entry.d/73-expand-templates.sh b/image-entry.d/73-expand-templates.sh index f6688f7..016f040 100755 --- a/image-entry.d/73-expand-templates.sh +++ b/image-entry.d/73-expand-templates.sh @@ -37,12 +37,26 @@ for n in ${NGX_DIRS_MERGE} ; do merge_dirs="${merge_dirs} $n/" done -expand_dir_envsubst ${merge_dirs} || expand_error - set -a J2CFG_PATH="${merged_root}/j2cfg" J2CFG_SEARCH_PATH="${merged_root}" -set -a +set +a + +## expand j2cfg templates first + +expand_dir_envsubst j2cfg/ || expand_error +expand_dir_j2cfg j2cfg/ || expand_error + +## expand other templates + +expand_dir_envsubst ${merge_dirs} || expand_error + +unset j2cfg_dump +j2cfg_dump="${volume_root}/diag.j2cfg.yml" + +j2cfg-dump > "${j2cfg_dump}" || expand_error + +export J2CFG_CONFIG="${j2cfg_dump}" expand_dir_j2cfg ${merge_dirs} || expand_error diff --git a/image-entry.d/74-combine-tree.sh b/image-entry.d/74-combine-tree.sh index 9f25126..efb4613 100755 --- a/image-entry.d/74-combine-tree.sh +++ b/image-entry.d/74-combine-tree.sh @@ -88,7 +88,7 @@ while read -r old_path ; do done <<-EOF $( set +e - for n in ${NGX_DIRS_MERGE} ; do + for n in j2cfg ${NGX_DIRS_MERGE} ; do [ -n "$n" ] || continue [ -d "${merged_root}/$n" ] || continue diff --git a/image-entry.d/75-remove-merged-tree.sh b/image-entry.d/75-remove-merged-tree.sh index 77b2495..640de27 100755 --- a/image-entry.d/75-remove-merged-tree.sh +++ b/image-entry.d/75-remove-merged-tree.sh @@ -3,6 +3,8 @@ set -f . /image-entry.d/00-common.envsh +IEP_RETAIN_MERGED_TREE=$(gobool_to_int "${IEP_RETAIN_MERGED_TREE:-0}" 0) + if [ "${IEP_RETAIN_MERGED_TREE}" = 1 ] ; then log_always "NOT removing merged tree: ${merged_root}/" else diff --git a/image-entry.d/90-angie-config-test.sh b/image-entry.d/90-angie-config-test.sh index 252e0bd..ba93009 100755 --- a/image-entry.d/90-angie-config-test.sh +++ b/image-entry.d/90-angie-config-test.sh @@ -6,18 +6,20 @@ set -f ## Angie: unset core variable unset ANGIE ANGIE_BPF_MAPS +_angie() { + angie -e stderr -g 'error_log /dev/stderr warn;' "$@" +} + ## merely debug test log_always 'test Angie configuration:' log_always '=========================' -angie -t +_angie -t r=$? log_always '=========================' -## cleanup after test -rm -f "${volume_root}/angie.pid" - if [ $r = 0 ] ; then log_always 'ready to run Angie' + _angie -T 2>&1 | cat > "${volume_root}/diag.angie.conf" else log_always 'configuration test has failed, see above' t=15 @@ -25,4 +27,7 @@ else sleep $t fi +## cleanup after test +rm -f "${volume_root}/angie.pid" + exit 0 diff --git a/image-entry.d/99-cleanup-env.envsh b/image-entry.d/99-cleanup-env.envsh index 1e95ea0..94a5963 100755 --- a/image-entry.d/99-cleanup-env.envsh +++ b/image-entry.d/99-cleanup-env.envsh @@ -3,6 +3,8 @@ ## Angie: unset core variable unset ANGIE ANGIE_BPF_MAPS +IEP_RETAIN_ENV=$(gobool_to_int "${IEP_RETAIN_ENV:-0}" 0) + if [ "${IEP_RETAIN_ENV}" = 1 ] ; then log_always "NOT removing following variables:" sed -E '/^./s,^, ,' >&2 diff --git a/image-entry.sh b/image-entry.sh index 2c18c00..9f928de 100755 --- a/image-entry.sh +++ b/image-entry.sh @@ -93,10 +93,6 @@ IEP_INIT=$(gobool_to_int "${IEP_INIT:-0}" 0) # unexport IEP_INIT unset x ; x="${IEP_INIT}" ; unset IEP_INIT ; IEP_INIT="$x" ; unset x -IEP_RETAIN_MERGED_TREE=$(gobool_to_int "${IEP_RETAIN_MERGED_TREE:-0}" 0) -IEP_RETAIN_ENV=$(gobool_to_int "${IEP_RETAIN_ENV:-0}" 0) -export IEP_RETAIN_MERGED_TREE IEP_RETAIN_ENV - # IEP_TRACE=$(gobool_to_int "${IEP_TRACE:-0}" 0) IEP_DEBUG=$(gobool_to_int "${IEP_DEBUG:-0}" 0) IEP_VERBOSE=$(gobool_to_int "${IEP_VERBOSE:-${IEP_DEBUG}}" "${IEP_DEBUG}") diff --git a/j2cfg/j2cfg-dump.py b/j2cfg/j2cfg-dump.py new file mode 100755 index 0000000..aaec791 --- /dev/null +++ b/j2cfg/j2cfg-dump.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import os.path +import sys + + +def main(): + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import j2cfg + + j = j2cfg.J2cfg(dump_only=True) + print(j.dump_config()) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/j2cfg/j2cfg/__init__.py b/j2cfg/j2cfg/__init__.py index dd68feb..720fb01 100644 --- a/j2cfg/j2cfg/__init__.py +++ b/j2cfg/j2cfg/__init__.py @@ -16,14 +16,16 @@ J2CFG_CONFIG_EXT = ['yml', 'yaml', 'json'] class J2cfg: - def __init__(self, strict=True, config=None, config_path=None, - modules=None, search_path=None, template_suffix=None): + def __init__(self, strict=True, config_file=None, config_path=None, + modules=None, search_path=None, template_suffix=None, + dump_only=False): - self.strict = strict - if not isinstance(self.strict, bool): - self.strict = True + if dump_only is None: + self.dump_only = False + else: + self.dump_only = bool(dump_only) - self.config_file = config or os.getenv('J2CFG_CONFIG') + self.config_file = config_file or os.getenv('J2CFG_CONFIG') if self.config_file is not None: self.config_file = str(self.config_file) @@ -38,6 +40,87 @@ class J2cfg: self.config_path = any_to_str_list(self.config_path) self.config_path = uniq_str_list(self.config_path) + self.kwargs = {'j2cfg': {}} + + def merge_dict_from_file(filename): + if filename is None: + return False + f = str(filename) + if f == '': + return False + if not os.path.exists(f): + return False + if not os.path.isfile(f): + print( + f'J2cfg: not a file, skipping: {filename}', + file=sys.stderr) + return False + + if f.endswith('.yml') or f.endswith('.yaml'): + with open(f, mode='r', encoding='utf-8') as fx: + for x in yaml.safe_load_all(fx): + if not x: + continue + self.kwargs['j2cfg'] = merge_dict_recurse( + self.kwargs['j2cfg'], x + ) + return True + + if f.endswith('.json'): + with open(f, mode='r', encoding='utf-8') as fx: + self.kwargs['j2cfg'] = merge_dict_recurse( + self.kwargs['j2cfg'], json.load(fx) + ) + return True + + print( + f'J2cfg: non-recognized name extension: {filename}', + file=sys.stderr) + return False + + def merge_dict_default(): + search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT]) + search_flags = wcmatch.wcmatch.SYMLINKS + + for d in self.config_path: + if not os.path.isdir(d): + continue + m = wcmatch.wcmatch.WcMatch(d, search_pattern, + flags=search_flags) + for f in sorted(m.match()): + if self.dump_only: + real_f = os.path.realpath(f) + if f == real_f: + print( + f'J2cfg: try loading {f}', + file=sys.stderr + ) + else: + print( + f'J2cfg: try loading {f} <- {real_f}', + file=sys.stderr + ) + merge_dict_from_file(f) + + if self.config_file is None: + merge_dict_default() + else: + if os.path.isfile(self.config_file): + merge_dict_from_file(self.config_file) + else: + print( + 'J2cfg: J2cfg config file does not exist, skipping: ' + + f'{self.config_file}', + file=sys.stderr + ) + + if self.dump_only: + return + + self.strict = strict + if not isinstance(self.strict, bool): + self.strict = True + self.search_path = search_path if self.search_path is None: self.search_path = os.getenv('J2CFG_SEARCH_PATH') @@ -65,20 +148,19 @@ class J2cfg: self.template_suffix = template_suffix or os.getenv('J2CFG_SUFFIX') if self.template_suffix is None: - self.template_suffix = J2CFG_TEMPLATE_EXT + self.template_suffix = '' + J2CFG_TEMPLATE_EXT else: self.template_suffix = str(self.template_suffix) if self.template_suffix == '': - self.template_suffix = J2CFG_TEMPLATE_EXT + self.template_suffix = '' + J2CFG_TEMPLATE_EXT if not self.template_suffix.startswith('.'): self.template_suffix = '.' + self.template_suffix - self.kwargs = { + self.kwargs.update({ 'env': os.environ, 'env_preserve': J2CFG_PRESERVE_ENVS.copy(), 'env_passthrough': J2CFG_PASSTHROUGH_ENVS.copy(), - 'j2cfg': {} - } + }) for m in self.modules: if m in self.kwargs: print(f'J2cfg: kwargs already has {m} key', @@ -86,61 +168,6 @@ class J2cfg: continue self.kwargs[m] = importlib.import_module(m) - def merge_dict_from_file(filename): - if filename is None: - return False - f = str(filename) - if f == '': - return False - if not os.path.exists(f): - return False - if not os.path.isfile(f): - print( - f'J2cfg: not a file, skipping: {filename}', - file=sys.stderr) - return False - - if f.endswith('.yml') or f.endswith('.yaml'): - with open(f, mode='r', encoding='utf-8') as fx: - x = yaml.safe_load(fx) - self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x - return True - - if f.endswith('.json'): - with open(f, mode='r', encoding='utf-8') as fx: - x = json.load(fx) - self.kwargs['j2cfg'] = self.kwargs['j2cfg'] | x - return True - - print( - f'J2cfg: non-recognized name extension: {filename}', - file=sys.stderr) - return False - - def merge_dict_default(): - search_pattern = '|'.join(['*.' + ext for ext in J2CFG_CONFIG_EXT]) - search_flags = wcmatch.wcmatch.SYMLINKS - - for d in self.config_path: - if not os.path.isdir(d): - continue - m = wcmatch.wcmatch.WcMatch(d, search_pattern, - flags=search_flags) - for f in sorted(m.match()): - merge_dict_from_file(f) - - if self.config_file is None: - merge_dict_default() - else: - if os.path.isfile(self.config_file): - merge_dict_from_file(self.config_file) - else: - print( - 'J2cfg: J2cfg config file does not exist, skipping: ' - + f'{self.config_file}', - file=sys.stderr - ) - self.j2fs_loaders = { d: jinja2.FileSystemLoader( d, encoding='utf-8', followlinks=True, @@ -164,7 +191,13 @@ class J2cfg: init_env(self.j2env) + def dump_config(self): + return yaml.safe_dump(self.kwargs['j2cfg']) + def ensure_fs_loader_for(self, directory: str): + if self.dump_only: + raise ValueError('dump_only is True') + if directory in self.j2fs_loaders: return self.j2fs_loaders[directory] = jinja2.FileSystemLoader( @@ -172,6 +205,8 @@ class J2cfg: ) def render_file(self, file_in, file_out=None) -> bool: + if self.dump_only: + raise ValueError('dump_only is True') def render_error(msg) -> bool: if self.strict: diff --git a/j2cfg/j2cfg/functions.py b/j2cfg/j2cfg/functions.py index 8972f9e..7645c74 100644 --- a/j2cfg/j2cfg/functions.py +++ b/j2cfg/j2cfg/functions.py @@ -2,6 +2,7 @@ import collections.abc import itertools import pathlib import re +import sys import jinja2 @@ -239,6 +240,41 @@ def sh_like_file_to_list(j2env, file_in: str) -> list: )) +def merge_dict_recurse(d1, d2: dict) -> dict: + x = {} | d1 + + keys1 = set(x.keys()) + keys2 = set(d2.keys()) + common = keys1 & keys2 + missing = keys2 - common + + map1 = {k for k in common if is_mapping(x.get(k))} + seq1 = {k for k in common if is_sequence(x.get(k))} + misc1 = common - seq1 - map1 + + merge_safe = missing | misc1 + x.update({k: d2.get(k) for k in merge_safe}) + + map_common = {k for k in map1 if is_mapping(d2.get(k))} + for k in map_common: + x[k] = merge_dict_recurse(x.get(k), d2.get(k)) + + seq_common = {k for k in seq1 if is_sequence(d2.get(k))} + for k in seq_common: + x[k] = uniq(list(x.get(k)) + list(d2.get(k))) + + unmerged = (map1 - map_common) | (seq1 - seq_common) + for k in unmerged: + t1 = type(x.get(k)) + t2 = type(d2.get(k)) + print( + f'merge_dict_recurse(): skipping key {k}' + + f' due to type mismatch: {t1} vs. {t2}', + file=sys.stderr) + + return x + + J2CFG_FILTERS = [ any_to_env_dict, any_to_str_list, diff --git a/scripts/angie-builtin-modules.sh b/scripts/angie-builtin-modules.sh index 2302e05..9321c93 100755 --- a/scripts/angie-builtin-modules.sh +++ b/scripts/angie-builtin-modules.sh @@ -2,17 +2,17 @@ set -f conf_dir='/etc/angie' -conf_file="${conf_dir}/.none.conf" -pid_file='/run/angie/none.pid' -angie -q -e /dev/stderr -g "error_log /dev/stderr warn; pid ${pid_file};" -c "${conf_file}" -t -r=$? -rm -f "${pid_file}" -[ $r -eq 0 ] || exit $r +## Angie: unset core variable +unset ANGIE ANGIE_BPF_MAPS + +_angie() { + angie -e stderr -g 'error_log /dev/stderr warn;' "$@" +} t=$(mktemp) || exit $? -angie -c "${conf_file}" -m 2>&1 | tee "$t" >/dev/null +_angie -m 2>&1 | tee "$t" >/dev/null sed -En '/^ngx_(http|mail|stream)/d;/^ngx_(.+)_module$/{s//\1/;s/_filter$//;s/_/-/g;p}' < "$t" \ | sort -uV > "${conf_dir}/builtin.core" diff --git a/scripts/j2cfg-dump b/scripts/j2cfg-dump new file mode 100755 index 0000000..d6685e9 --- /dev/null +++ b/scripts/j2cfg-dump @@ -0,0 +1,2 @@ +#!/bin/sh +exec python3 "/usr/local/lib/j2cfg/${0##*/}.py" "$@" \ No newline at end of file