From fa7852d5a53279c4e7918d6972fee2082f4aea4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Barnouin?= Date: Tue, 13 May 2025 13:50:05 +0200 Subject: [PATCH 1/4] Send logs to promtail directly (not rsyslog) --- systems/minimalLXCConfig.nix | 2 +- systems/minimalMicrovmConfig.nix | 2 +- systems/minimalVMConfig.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/systems/minimalLXCConfig.nix b/systems/minimalLXCConfig.nix index 9de7c03..ad71387 100644 --- a/systems/minimalLXCConfig.nix +++ b/systems/minimalLXCConfig.nix @@ -128,7 +128,7 @@ }; rsyslogd = { enable = true; - extraConfig = "*.*@192.168.1.27:514;RSYSLOG_SyslogProtocol23Format"; + extraConfig = "*.*@192.168.1.27:1514;RSYSLOG_SyslogProtocol23Format"; }; prometheus = { exporters = { diff --git a/systems/minimalMicrovmConfig.nix b/systems/minimalMicrovmConfig.nix index 0284417..e0bc8d5 100644 --- a/systems/minimalMicrovmConfig.nix +++ b/systems/minimalMicrovmConfig.nix @@ -181,7 +181,7 @@ in { }; rsyslogd = { enable = true; - extraConfig = "*.*@192.168.1.27:514;RSYSLOG_SyslogProtocol23Format"; + extraConfig = "*.*@192.168.1.27:1514;RSYSLOG_SyslogProtocol23Format"; }; prometheus = { exporters = { diff --git a/systems/minimalVMConfig.nix b/systems/minimalVMConfig.nix index 8eeb24e..c8cf91b 100644 --- a/systems/minimalVMConfig.nix +++ b/systems/minimalVMConfig.nix @@ -115,7 +115,7 @@ }; rsyslogd = { enable = true; - extraConfig = "*.*@192.168.1.27:514;RSYSLOG_SyslogProtocol23Format"; + extraConfig = "*.*@192.168.1.27:1514;RSYSLOG_SyslogProtocol23Format"; }; prometheus = { exporters = { From 54ec8c85d36795cc4cdde6411fbb77212e6c6845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Barnouin?= Date: Tue, 13 May 2025 13:50:26 +0200 Subject: [PATCH 2/4] Improve Loki config --- services/grafana/default.nix | 95 ++++++++++++------------------------ 1 file changed, 30 insertions(+), 65 deletions(-) diff --git a/services/grafana/default.nix b/services/grafana/default.nix index 172ff0e..01386de 100644 --- a/services/grafana/default.nix +++ b/services/grafana/default.nix @@ -55,20 +55,6 @@ in { ]; }; }; - rsyslogd = { - enable = true; - extraConfig = '' - ruleset(name="remote"){ - action(type="omfwd" Target="localhost" Port="1514" Protocol="tcp" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted") - } - - module(load="imudp") - input(type="imudp" port="514" ruleset="remote") - - module(load="imtcp") - input(type="imtcp" port="514" ruleset="remote") - ''; - }; influxdb2 = { enable = true; }; @@ -270,59 +256,20 @@ in { loki = { enable = true; configuration = { - server.http_listen_port = 3100; - server.grpc_listen_port = 9096; auth_enabled = false; - ingester = { - lifecycler = { - address = "192.168.1.27"; - ring = { - kvstore = { - store = "inmemory"; - }; - replication_factor = 1; - }; + server = { + http_listen_port = 3100; + grpc_listen_port = 9096; + }; + common = { + instance_addr = "0.0.0.0"; + path_prefix = "/tmp/loki"; + storage.filesystem = { + chunks_directory = "/tmp/loki/chunks"; + rules_directory = "/tmp/loki/rules"; }; - chunk_idle_period = "1h"; - max_chunk_age = "1h"; - chunk_target_size = 999999; - chunk_retain_period = "30s"; - }; - schema_config = { - configs = [ - { - from = "2022-06-06"; - store = "boltdb-shipper"; - object_store = "filesystem"; - schema = "v13"; - index = { - prefix = "index_"; - period = "24h"; - }; - } - ]; - }; - storage_config = { - boltdb_shipper = { - active_index_directory = "/var/lib/loki/boltdb-shipper-active"; - cache_location = "/var/lib/loki/boltdb-shipper-cache"; - cache_ttl = "24h"; - }; - - filesystem = { - directory = "/var/lib/loki/chunks"; - }; - }; - - limits_config = { - reject_old_samples = true; - reject_old_samples_max_age = "168h"; - allow_structured_metadata = false; - }; - - table_manager = { - retention_deletes_enabled = false; - retention_period = "0s"; + replication_factor = 1; + ring.kvstore.store = "inmemory"; }; compactor = { working_directory = "/var/lib/loki"; @@ -332,6 +279,24 @@ in { }; }; }; + query_range.results_cache.cache.embedded_cache = { + enabled = true; + max_size_mb = 100; + }; + schema_config.configs = [ + { + from = "2025-05-13"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + ruler.alertmanager_url = "http://localhost:9093"; + analytics.reporting_enabled = false; }; }; promtail = { From 0511396ca71291c7becd990818f1f129a2edca69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Barnouin?= Date: Tue, 13 May 2025 14:15:43 +0200 Subject: [PATCH 3/4] Improve rsyslog config --- systems/minimalLXCConfig.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/systems/minimalLXCConfig.nix b/systems/minimalLXCConfig.nix index ad71387..2834477 100644 --- a/systems/minimalLXCConfig.nix +++ b/systems/minimalLXCConfig.nix @@ -128,7 +128,9 @@ }; rsyslogd = { enable = true; - extraConfig = "*.*@192.168.1.27:1514;RSYSLOG_SyslogProtocol23Format"; + extraConfig = '' + *.* action(type="omfwd" target="192.168.1.27" port="1514" protocol="tcp") + ''; }; prometheus = { exporters = { From 3fd9c73fc1291bd99b011f725f229baba2094c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Barnouin?= Date: Tue, 13 May 2025 14:16:14 +0200 Subject: [PATCH 4/4] Update flake --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 8658e30..889ccc4 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1741508717, - "narHash": "sha256-iQf1WdNxaApOFHIx4RLMRZ4f8g+8Xp0Z1/E/Mz2rLxY=", + "lastModified": 1744897914, + "narHash": "sha256-GIVU92o2TZBnKQXTb76zpQbWR4zjU2rFqWKNIIpXnqA=", "owner": "yaxitech", "repo": "ragenix", - "rev": "2a2bea99d74927e54adf53cbf113219def67d5c9", + "rev": "40f2e17ecaeab4d78ec323e96a04548c0aaa5223", "type": "github" }, "original": { @@ -169,11 +169,11 @@ ] }, "locked": { - "lastModified": 1744117652, - "narHash": "sha256-t7dFCDl4vIOOUMhEZnJF15aAzkpaup9x4ZRGToDFYWI=", + "lastModified": 1747020534, + "narHash": "sha256-D/6rkiC6w2p+4SwRiVKrWIeYzun8FBg7NlMKMwQMxO0=", "owner": "nix-community", "repo": "home-manager", - "rev": "b4e98224ad1336751a2ac7493967a4c9f6d9cb3f", + "rev": "b4bbdc6fde16fc2051fcde232f6e288cd22007ca", "type": "github" }, "original": { @@ -201,11 +201,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1744309437, - "narHash": "sha256-QZnNHM823am8apCqKSPdtnzPGTy2ZB4zIXOVoBp5+W0=", + "lastModified": 1746957726, + "narHash": "sha256-k9ut1LSfHCr0AW82ttEQzXVCqmyWVA5+SHJkS5ID/Jo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f9ebe33a928b5d529c895202263a5ce46bdf12f7", + "rev": "a39ed32a651fdee6842ec930761e31d1f242cb94", "type": "github" }, "original": {