From 7d91e94fa98dc96933970d37a335e2aba2e72d5d Mon Sep 17 00:00:00 2001 From: Jonni Liljamo Date: Mon, 2 Sep 2024 20:52:29 +0300 Subject: [PATCH] feat: add metrics --- .sops.yaml | 7 ++ hosts/metrics/default.nix | 17 +++ secrets/metrics/secrets.yaml | 34 ++++++ systems/hosts/default.nix | 5 + systems/hosts/metrics/default.nix | 193 ++++++++++++++++++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 hosts/metrics/default.nix create mode 100644 secrets/metrics/secrets.yaml create mode 100644 systems/hosts/metrics/default.nix diff --git a/.sops.yaml b/.sops.yaml index 81d6a38..0705802 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -3,6 +3,7 @@ keys: - &arwen age15hcszwfk0d6cu9ua6g4udj9tdq63jm8lja66ktxu0fjfuczczcwsm5kcxn - &alice age1pqjj62u9u3x658a5u47nf7uf0cfek2ht09ztqamjfl7j8s2xeduqx5cfnn - &dns age1m5ktjargxxu04dn9c2uhvaw79z74mxsc4vdrkalxjn4aa8c86plqg0hyyw + - &metrics age1m8u3a7rzyx2n6zjxjnfkla34yk3v77egxzd3lv9umt69lsynlaqqqfpt05 - &sqbuilds age1wgzza5upq4tcpanmx3p9tg9swltz58ycufcapq9s45wpq8mtvepsr0lnzk creation_rules: - path_regex: secrets/arwen/[^/]+\.yaml$ @@ -23,6 +24,12 @@ creation_rules: - *liljamo_gpg age: - *dns + - path_regex: secrets/metrics/[^/]+\.yaml$ + key_groups: + - pgp: + - *liljamo_gpg + age: + - *metrics - path_regex: secrets/sqbuilds/[^/]+\.yaml$ key_groups: - pgp: diff --git a/hosts/metrics/default.nix b/hosts/metrics/default.nix new file mode 100644 index 0000000..d840393 --- /dev/null +++ b/hosts/metrics/default.nix @@ -0,0 +1,17 @@ +{config, ...}: { + sops.secrets.rootPwd.neededForUsers = true; + sops.secrets.liljamoPwd.neededForUsers = true; + + roles.base = { + root.hashedPasswordFile = config.sops.secrets.rootPwd.path; + primaryUser = { + username = "liljamo"; + hashedPasswordFile = config.sops.secrets.liljamoPwd.path; + }; + }; + + roles.tailscale = { + enable = true; + enableSSH = true; + }; +} diff --git a/secrets/metrics/secrets.yaml b/secrets/metrics/secrets.yaml new file mode 100644 index 0000000..dbc54dc --- /dev/null +++ b/secrets/metrics/secrets.yaml @@ -0,0 +1,34 @@ +rootPwd: ENC[AES256_GCM,data:qtSJNQZaN/++KhOoBnyaAyovBMoH+kawjGAGWqShiQ6OkJ3xNpNxoCoGxRpnjvRLejzIdQrKaUyNcRBFRCQhjci0hRREpPzOATH5I2LTr6QvqxN+yZnQjzpD88MWzfgiferGKgo8jZ9Iig==,iv:t/7R3Ox91Ogplrol+/aOTDqHaNDKyB8k52gN40dcUOc=,tag:qln+E2gIFoXKE87d4yXuKw==,type:str] +liljamoPwd: ENC[AES256_GCM,data:3mPe9sLoPGQQ8xybO3eO5wuuBaKBtSD70Spn9MOgkZMkbxKqebOBY0hiTeHFGMGRpYanlm7rYUefVudLQEBIhb7FP0YDDsrWeRGZxKJnxR0I8PyL75A2Einc9+gmnlT8q0pbxZTFN3Zw2g==,iv:FFEb304O/SpZrRYgAhGaOwqpKN1Pbch2KLhx0DVAMHE=,tag:40WSr5J0mKNOgTpFyso5SQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1m8u3a7rzyx2n6zjxjnfkla34yk3v77egxzd3lv9umt69lsynlaqqqfpt05 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuUFRJZHNkQUpjNnNONWNq + RWt1SFJnMm5NUko0QS8wblhWMGF1RGhWSnpBCkQvNFpKT1pQNisvV0wxMURjNmRv + OXNUL3RpZnducVZycld4YW9FQWw4WnMKLS0tIFAzaXNmcjJkVElEMzNaRDIyMEJO + VEI4TjdpcGI2a0d2VThQelUxVUFCNDgKNRElA7Bd0KmMKWJs/VSzT2rdImYn9EyS + 5RUHFPXKTwPOY9TMcFvah2b5j9VZNeK7PWOp9YeGtMObgdS7l855Mg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-09-02T17:09:32Z" + mac: ENC[AES256_GCM,data:EgkxyzvB9WiEaPbVxe2hIAgKQLvxNlJXnKfLDtsUvpS5B+yjgcEV/z20c5WfdMX2IxHvA13+pg7Bg0DAP7yqxavrUK7rOje/iHR0H5CgdbwoYRRRIs97hxQT+OFxmqOum1yKl1HtQayZmDNBu6P6Nvd40Fff5NP2iYQR5QgOKxI=,iv:TWbhEy7vWnBLgJi6qht3FulE/cpYwWJWV3fgwdui7rY=,tag:Yz1z1QtIvWwh3wHcDZ8s6g==,type:str] + pgp: + - created_at: "2024-09-02T17:08:27Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4D8ab0ENzkR4wSAQdAXGXGCASUFLyKYDHP9h2KqeiBuKO1kKW9Z5hjVHm9PHIw + TV5awurWOu+oU7ktjs1YC6TI2RP6vQP9OVrLdelVNIWUFyZpQhScZyF+j6TB3lvp + 1GgBCQIQarehOo/AqoFhTqFYtBV3iknvLfErMUopYYQksw0hM6DclVe56LhZVPnw + 274FhcmmJLYHH/9pApBCDDyrpccJdT3tsNGX6I7IvxLkRUQ60ubXufBB2mZREpgi + UvxDo8Y6algxcg== + =2YyN + -----END PGP MESSAGE----- + fp: 848EEBCEE9F0D29D25C321A658577946A65EB712 + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/systems/hosts/default.nix b/systems/hosts/default.nix index d95da8c..4aa0c96 100644 --- a/systems/hosts/default.nix +++ b/systems/hosts/default.nix @@ -22,6 +22,11 @@ profile = lxc; modules = []; }; + metrics = { + system = "x86_64-linux"; + profile = lxc; + modules = []; + }; # VMs sqbuilds = { diff --git a/systems/hosts/metrics/default.nix b/systems/hosts/metrics/default.nix new file mode 100644 index 0000000..44201e1 --- /dev/null +++ b/systems/hosts/metrics/default.nix @@ -0,0 +1,193 @@ +{...}: let + influxDB2Port = 8086; + prometheusPort = 9090; + lokiPort = 9091; + grafanaPort = 3000; +in { + networking.firewall.allowedTCPPorts = [ + influxDB2Port + prometheusPort + ]; + + services.influxdb2 = { + enable = true; + settings = { + http-bind-address = ":${toString influxDB2Port}"; + }; + }; + + services.prometheus = { + enable = true; + port = prometheusPort; + globalConfig = { + scrape_interval = "5s"; + }; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { + targets = ["localhost:${toString prometheusPort}"]; + } + ]; + } + + # lxcmetrics + # node, systemd + { + job_name = "lxcmetrics_job"; + static_configs = [ + { + targets = ["localhost:9100" "localhost:9558"]; + } + ]; + } + + # lxchydra + # node, systemd + { + job_name = "lxchydra_job"; + static_configs = [ + { + targets = ["10.1.2.2:9100" "10.1.2.2:9558"]; + } + ]; + } + + # lxcproxy + # haproxy, node, systemd + { + job_name = "lxcproxy_job"; + static_configs = [ + { + targets = ["10.1.2.10:8404" "10.1.2.10:9100" "10.1.2.10:9558"]; + } + ]; + } + + # lxccloud + # node, systemd + { + job_name = "lxccloud_job"; + static_configs = [ + { + targets = ["10.1.2.15:9100" "10.1.2.15:9558"]; + } + ]; + } + + # uwulpine vm + { + job_name = "node_uwulpine"; + static_configs = [ + { + targets = ["10.1.1.10:9091"]; + } + ]; + } + { + job_name = "cadvisor_uwulpine"; + static_configs = [ + { + targets = ["10.1.1.10:9092"]; + } + ]; + } + { + job_name = "jellyfin"; + static_configs = [ + { + targets = ["10.1.2.20:8096"]; + } + ]; + } + ]; + }; + + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server.http_listen_port = lokiPort; + + ingester = { + lifecycler = { + address = "0.0.0.0"; + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 1048576; + chunk_retain_period = "30s"; + }; + + schema_config = { + configs = [ + { + from = "2022-06-06"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + }; + + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + allow_structured_metadata = false; + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + max_query_series = 5000; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + + compactor = { + working_directory = "/var/lib/loki"; + compactor_ring = { + kvstore = { + store = "inmemory"; + }; + }; + }; + }; + }; + + services.grafana = { + enable = true; + settings = { + server = { + http_port = grafanaPort; + http_addr = "0.0.0.0"; + }; + "auth.anonymous".enabled = true; + security.allow_embedding = true; + }; + }; + + system.stateVersion = "24.05"; +} -- 2.44.1