From ecbdd9c2b4e5571354ce1de5cde30f468844768b Mon Sep 17 00:00:00 2001 From: James Greenhill Date: Fri, 17 Nov 2023 11:38:51 -0800 Subject: [PATCH] chore: add throw_on_max_partitions_per_insert_block=false to dev config (#18729) --- docker/clickhouse/config.xml | 284 +++++++++++++++++++++----------- docker/clickhouse/users-dev.xml | 35 ++-- docker/clickhouse/users.xml | 35 ++-- 3 files changed, 231 insertions(+), 123 deletions(-) diff --git a/docker/clickhouse/config.xml b/docker/clickhouse/config.xml index f3f858be7d1..31a58956f6e 100644 --- a/docker/clickhouse/config.xml +++ b/docker/clickhouse/config.xml @@ -20,17 +20,20 @@ - trace - test (not for production usage) - [1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114 + [1]: + https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114 --> trace /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 1000M 10 - + - + @@ -217,7 +225,8 @@ /path/to/ssl_ca_cert_file - none @@ -232,10 +241,12 @@ false - + - + /etc/clickhouse-server/server.crt /etc/clickhouse-server/server.key + true true sslv2,sslv3 @@ -264,24 +276,30 @@ - + 100 0 @@ -302,21 +320,25 @@ --> 0.9 - 4194304 - 0 - @@ -341,14 +363,18 @@ - - - + true @@ -644,14 +698,16 @@ - + localhost 9000 - + @@ -666,22 +722,28 @@ Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts. If port is explicitly specified in URL, the host:port is checked as a whole. If host specified here without port, any port with this host allowed. - "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed. - If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]". - If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked. + "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only + "yandex.ru:80" is allowed. + If the host is specified as IP address, it is checked as specified in URL. Example: + "[2a02:6b8:a::a]". + If there are redirects and support for redirects is enabled, every redirect (the Location field) is + checked. Host should be specified using the host xml tag: yandex.ru --> .* - @@ -701,7 +763,8 @@ @@ -710,7 +773,6 @@ - 3600 @@ -788,7 +850,8 @@ system query_log
toYYYYMM(event_date) - @@ -843,7 +909,8 @@ + Part log contains information about all actions with parts in MergeTree tables (creation, deletion, + merges, downloads).--> system part_log
@@ -852,8 +919,10 @@
- + system metric_log
@@ -933,7 +1003,8 @@ --> - + @@ -965,12 +1036,14 @@ --> - - - + + + - - - - - - + + + + + + - - - + + + - - + + @@ -1032,16 +1107,21 @@ - + + 0 + /var/lib/clickhouse/format_schemas/ - false - + false - + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 @@ -1183,4 +1269,4 @@ --> - + \ No newline at end of file diff --git a/docker/clickhouse/users-dev.xml b/docker/clickhouse/users-dev.xml index dd6e54d7c5d..704e99ef9e9 100644 --- a/docker/clickhouse/users-dev.xml +++ b/docker/clickhouse/users-dev.xml @@ -15,7 +15,8 @@ with minimum number of different symbols between replica's hostname and local hostname (Hamming distance). in_order - first live replica is chosen in specified order. - first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors. + first_or_random - if first replica one has higher number of errors, pick a random one from replicas + with minimum number of errors. --> random @@ -45,30 +46,39 @@ Password could be empty. If you want to specify SHA256, place it in 'password_sha256_hex' element. - Example: 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5 - Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019). + Example: + 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5 + Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July + 2019). If you want to specify double SHA1, place it in 'password_double_sha1_hex' element. - Example: e395796d6546b1b65db9d665cd43f0e858dd4303 + Example: + e395796d6546b1b65db9d665cd43f0e858dd4303 - If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication, + If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for + authentication, place its name in 'server' element inside 'ldap' element. Example: my_ldap_server - If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config), + If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in + the main config), place 'kerberos' element instead of 'password' (and similar) elements. - The name part of the canonical principal name of the initiator must match the user name for authentication to succeed. - You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests + The name part of the canonical principal name of the initiator must match the user name for + authentication to succeed. + You can also place 'realm' element inside 'kerberos' element to further restrict authentication to + only those requests whose initiator's realm matches it. Example: Example: EXAMPLE.COM How to generate decent password: - Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | + sha256sum | tr -d '-' In first line will be password and in second - corresponding SHA256. How to generate double SHA1: - Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | + sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' In first line will be password and in second - corresponding double SHA1. --> @@ -89,7 +99,8 @@ To check access, DNS query is performed, and all received addresses compared to peer address. Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$ To check access, DNS PTR query is performed for peer address and then regexp is applied. - Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address. + Then, for result of PTR query, another DNS query is performed and all received addresses compared + to peer address. Strongly recommended that regexp is ends with $ All results of DNS requests are cached till server restart. --> @@ -126,4 +137,4 @@ - + \ No newline at end of file diff --git a/docker/clickhouse/users.xml b/docker/clickhouse/users.xml index 49ac9f73e0d..ece3df0f09f 100644 --- a/docker/clickhouse/users.xml +++ b/docker/clickhouse/users.xml @@ -15,7 +15,8 @@ with minimum number of different symbols between replica's hostname and local hostname (Hamming distance). in_order - first live replica is chosen in specified order. - first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors. + first_or_random - if first replica one has higher number of errors, pick a random one from replicas + with minimum number of errors. --> random @@ -43,30 +44,39 @@ Password could be empty. If you want to specify SHA256, place it in 'password_sha256_hex' element. - Example: 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5 - Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019). + Example: + 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5 + Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July + 2019). If you want to specify double SHA1, place it in 'password_double_sha1_hex' element. - Example: e395796d6546b1b65db9d665cd43f0e858dd4303 + Example: + e395796d6546b1b65db9d665cd43f0e858dd4303 - If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication, + If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for + authentication, place its name in 'server' element inside 'ldap' element. Example: my_ldap_server - If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config), + If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in + the main config), place 'kerberos' element instead of 'password' (and similar) elements. - The name part of the canonical principal name of the initiator must match the user name for authentication to succeed. - You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests + The name part of the canonical principal name of the initiator must match the user name for + authentication to succeed. + You can also place 'realm' element inside 'kerberos' element to further restrict authentication to + only those requests whose initiator's realm matches it. Example: Example: EXAMPLE.COM How to generate decent password: - Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | + sha256sum | tr -d '-' In first line will be password and in second - corresponding SHA256. How to generate double SHA1: - Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | + sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' In first line will be password and in second - corresponding double SHA1. --> @@ -87,7 +97,8 @@ To check access, DNS query is performed, and all received addresses compared to peer address. Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$ To check access, DNS PTR query is performed for peer address and then regexp is applied. - Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address. + Then, for result of PTR query, another DNS query is performed and all received addresses compared + to peer address. Strongly recommended that regexp is ends with $ All results of DNS requests are cached till server restart. --> @@ -124,4 +135,4 @@ - + \ No newline at end of file