bvm-logger: add custom clickhouse config
Just make it less spammy into the journal, sheesh.
This commit is contained in:
parent
086f5fe597
commit
37e36418a1
2 changed files with 193 additions and 0 deletions
185
ops/nixos/bvm-logger/clickhouse-config.xml
Normal file
185
ops/nixos/bvm-logger/clickhouse-config.xml
Normal file
|
|
@ -0,0 +1,185 @@
|
||||||
|
<?xml version="1.0"?>
|
||||||
|
<yandex>
|
||||||
|
<logger>
|
||||||
|
<!-- Possible levels [1]:
|
||||||
|
|
||||||
|
- none (turns off logging)
|
||||||
|
- fatal
|
||||||
|
- critical
|
||||||
|
- error
|
||||||
|
- warning
|
||||||
|
- notice
|
||||||
|
- information
|
||||||
|
- debug
|
||||||
|
- trace
|
||||||
|
|
||||||
|
[1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
|
||||||
|
-->
|
||||||
|
<level>critical</level>
|
||||||
|
<console>1</console>
|
||||||
|
<size>1000M</size>
|
||||||
|
<count>10</count>
|
||||||
|
</logger>
|
||||||
|
|
||||||
|
<http_port>8123</http_port>
|
||||||
|
<tcp_port>9000</tcp_port>
|
||||||
|
<interserver_http_port>9009</interserver_http_port>
|
||||||
|
|
||||||
|
<listen_host>::</listen_host>
|
||||||
|
|
||||||
|
<max_connections>4096</max_connections>
|
||||||
|
|
||||||
|
<keep_alive_timeout>3</keep_alive_timeout>
|
||||||
|
|
||||||
|
<max_concurrent_queries>100</max_concurrent_queries>
|
||||||
|
<max_server_memory_usage>0</max_server_memory_usage>
|
||||||
|
<max_thread_pool_size>10000</max_thread_pool_size>
|
||||||
|
<max_server_memory_usage_to_ram_ratio>0.9</max_server_memory_usage_to_ram_ratio>
|
||||||
|
<total_memory_profiler_step>4194304</total_memory_profiler_step>
|
||||||
|
<total_memory_tracker_sample_probability>0</total_memory_tracker_sample_probability>
|
||||||
|
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||||
|
<mark_cache_size>5368709120</mark_cache_size>
|
||||||
|
<mmap_cache_size>1000</mmap_cache_size>
|
||||||
|
<compiled_expression_cache_size>134217728</compiled_expression_cache_size>
|
||||||
|
<compiled_expression_cache_elements_size>10000</compiled_expression_cache_elements_size>
|
||||||
|
<path>/var/lib/clickhouse/</path>
|
||||||
|
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
||||||
|
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
||||||
|
<user_directories>
|
||||||
|
<users_xml>
|
||||||
|
<!-- Path to configuration file with predefined users. -->
|
||||||
|
<path>users.xml</path>
|
||||||
|
</users_xml>
|
||||||
|
<local_directory>
|
||||||
|
<!-- Path to folder where users created by SQL commands are stored. -->
|
||||||
|
<path>/var/lib/clickhouse/access/</path>
|
||||||
|
</local_directory>
|
||||||
|
</user_directories>
|
||||||
|
<default_profile>default</default_profile>
|
||||||
|
<custom_settings_prefixes></custom_settings_prefixes>
|
||||||
|
<default_database>default</default_database>
|
||||||
|
<mlock_executable>true</mlock_executable>
|
||||||
|
<remap_executable>false</remap_executable>
|
||||||
|
<builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
|
||||||
|
<max_session_timeout>3600</max_session_timeout>
|
||||||
|
<default_session_timeout>60</default_session_timeout>
|
||||||
|
|
||||||
|
<!-- Serve endpoint for Prometheus monitoring. -->
|
||||||
|
<!--
|
||||||
|
endpoint - mertics path (relative to root, statring with "/")
|
||||||
|
port - port to setup server. If not defined or 0 than http_port used
|
||||||
|
metrics - send data from table system.metrics
|
||||||
|
events - send data from table system.events
|
||||||
|
asynchronous_metrics - send data from table system.asynchronous_metrics
|
||||||
|
status_info - send data from different component from CH, ex: Dictionaries status
|
||||||
|
-->
|
||||||
|
<prometheus>
|
||||||
|
<endpoint>/metrics</endpoint>
|
||||||
|
<port>0</port>
|
||||||
|
|
||||||
|
<metrics>true</metrics>
|
||||||
|
<events>true</events>
|
||||||
|
<asynchronous_metrics>true</asynchronous_metrics>
|
||||||
|
<status_info>true</status_info>
|
||||||
|
</prometheus>
|
||||||
|
|
||||||
|
<query_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>query_log</table>
|
||||||
|
<partition_by>toYYYYMM(event_date)</partition_by>
|
||||||
|
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
|
||||||
|
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||||
|
</query_log>
|
||||||
|
<text_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>text_log</table>
|
||||||
|
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||||
|
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
|
||||||
|
<level>trace</level>
|
||||||
|
</text_log>
|
||||||
|
<trace_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>trace_log</table>
|
||||||
|
|
||||||
|
<partition_by>toYYYYMM(event_date)</partition_by>
|
||||||
|
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
|
||||||
|
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||||
|
</trace_log>
|
||||||
|
<query_thread_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>query_thread_log</table>
|
||||||
|
<partition_by>toYYYYMM(event_date)</partition_by>
|
||||||
|
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
|
||||||
|
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||||
|
</query_thread_log>
|
||||||
|
<metric_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>metric_log</table>
|
||||||
|
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||||
|
<collect_interval_milliseconds>1000</collect_interval_milliseconds>
|
||||||
|
</metric_log>
|
||||||
|
<asynchronous_metric_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>asynchronous_metric_log</table>
|
||||||
|
<flush_interval_milliseconds>7000</flush_interval_milliseconds>
|
||||||
|
</asynchronous_metric_log>
|
||||||
|
|
||||||
|
<crash_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>crash_log</table>
|
||||||
|
|
||||||
|
<partition_by />
|
||||||
|
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
||||||
|
</crash_log>
|
||||||
|
|
||||||
|
<!-- Custom TLD lists.
|
||||||
|
Format: <name>/path/to/file</name>
|
||||||
|
|
||||||
|
Changes will not be applied w/o server restart.
|
||||||
|
Path to the list is under top_level_domains_path (see above).
|
||||||
|
-->
|
||||||
|
<top_level_domains_lists>
|
||||||
|
<!--
|
||||||
|
<public_suffix_list>/path/to/public_suffix_list.dat</public_suffix_list>
|
||||||
|
-->
|
||||||
|
</top_level_domains_lists>
|
||||||
|
|
||||||
|
<!-- Configuration of external dictionaries. See:
|
||||||
|
https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts
|
||||||
|
-->
|
||||||
|
<dictionaries_config>*_dictionary.xml</dictionaries_config>
|
||||||
|
|
||||||
|
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
|
||||||
|
The directory will be created if it doesn't exist.
|
||||||
|
-->
|
||||||
|
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
|
||||||
|
|
||||||
|
<!-- Default query masking rules, matching lines would be replaced with something else in the logs
|
||||||
|
(both text logs and system.query_log).
|
||||||
|
name - name for the rule (optional)
|
||||||
|
regexp - RE2 compatible regular expression (mandatory)
|
||||||
|
replace - substitution string for sensitive data (optional, by default - six asterisks)
|
||||||
|
-->
|
||||||
|
<query_masking_rules>
|
||||||
|
<rule>
|
||||||
|
<name>hide encrypt/decrypt arguments</name>
|
||||||
|
<regexp>((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)</regexp>
|
||||||
|
<!-- or more secure, but also more invasive:
|
||||||
|
(aes_\w+)\s*\(.*\)
|
||||||
|
-->
|
||||||
|
<replace>\1(???)</replace>
|
||||||
|
</rule>
|
||||||
|
</query_masking_rules>
|
||||||
|
|
||||||
|
<send_crash_reports>
|
||||||
|
<!-- Changing <enabled> to true allows sending crash reports to -->
|
||||||
|
<!-- the ClickHouse core developers team via Sentry https://sentry.io -->
|
||||||
|
<!-- Doing so at least in pre-production environments is highly appreciated -->
|
||||||
|
<enabled>false</enabled>
|
||||||
|
<!-- Change <anonymize> to true if you don't feel comfortable attaching the server hostname to the crash report -->
|
||||||
|
<anonymize>false</anonymize>
|
||||||
|
<!-- Default endpoint should be changed to different Sentry DSN only if you have -->
|
||||||
|
<!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
|
||||||
|
<endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
|
||||||
|
</send_crash_reports>
|
||||||
|
</yandex>
|
||||||
|
|
@ -29,6 +29,14 @@ in {
|
||||||
my.ip.tailscale = "100.68.134.82";
|
my.ip.tailscale = "100.68.134.82";
|
||||||
|
|
||||||
services.clickhouse.enable = true;
|
services.clickhouse.enable = true;
|
||||||
|
services.clickhouse.package = pkgs.symlinkJoin {
|
||||||
|
name = "clickhouse-overridden";
|
||||||
|
paths = [ pkgs.clickhouse ];
|
||||||
|
postBuild = ''
|
||||||
|
rm $out/etc/clickhouse-server/config.xml
|
||||||
|
cp ${./clickhouse-config.xml} $out/etc/clickhouse-server/config.xml
|
||||||
|
'';
|
||||||
|
};
|
||||||
time.timeZone = "Etc/UTC";
|
time.timeZone = "Etc/UTC";
|
||||||
|
|
||||||
systemd.services.journal2clickhouse = {
|
systemd.services.journal2clickhouse = {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue