{...}: let
  influxDB2Port = 8086;
  prometheusPort = 9090;
  lokiPort = 9091;
  grafanaPort = 3000;
in {
  networking.firewall.allowedTCPPorts = [
    influxDB2Port
    prometheusPort
    lokiPort
    grafanaPort
  ];

  services.influxdb2 = {
    enable = true;
    settings = {
      http-bind-address = ":${toString influxDB2Port}";
    };
  };

  services.prometheus = {
    enable = true;
    port = prometheusPort;
    globalConfig = {
      scrape_interval = "5s";
    };
    scrapeConfigs = [
      {
        job_name = "prometheus";
        static_configs = [
          {
            targets = ["localhost:${toString prometheusPort}"];
          }
        ];
      }

      # lxcmetrics
      # node, systemd
      {
        job_name = "lxcmetrics_job";
        static_configs = [
          {
            targets = ["localhost:9100" "localhost:9558"];
          }
        ];
      }

      # lxcproxy
      # haproxy, node, systemd
      {
        job_name = "lxcproxy_job";
        static_configs = [
          {
            targets = ["10.1.2.10:8404" "10.1.2.10:9100" "10.1.2.10:9558"];
          }
        ];
      }

      # lxccloud
      # node, systemd
      {
        job_name = "lxccloud_job";
        static_configs = [
          {
            targets = ["10.1.2.15:9100" "10.1.2.15:9558"];
          }
        ];
      }

      # uwulpine vm
      {
        job_name = "node_uwulpine";
        static_configs = [
          {
            targets = ["10.1.1.10:9091"];
          }
        ];
      }
      {
        job_name = "cadvisor_uwulpine";
        static_configs = [
          {
            targets = ["10.1.1.10:9092"];
          }
        ];
      }
      {
        job_name = "jellyfin";
        static_configs = [
          {
            targets = ["10.1.2.20:8096"];
          }
        ];
      }
    ];
  };

  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      server.http_listen_port = lokiPort;

      ingester = {
        lifecycler = {
          address = "0.0.0.0";
          ring = {
            kvstore = {
              store = "inmemory";
            };
            replication_factor = 1;
          };
          final_sleep = "0s";
        };
        chunk_idle_period = "1h";
        max_chunk_age = "1h";
        chunk_target_size = 1048576;
        chunk_retain_period = "30s";
      };

      schema_config = {
        configs = [
          {
            from = "2022-06-06";
            store = "boltdb-shipper";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };

      storage_config = {
        boltdb_shipper = {
          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
          cache_location = "/var/lib/loki/boltdb-shipper-cache";
          cache_ttl = "24h";
        };

        filesystem = {
          directory = "/var/lib/loki/chunks";
        };
      };

      limits_config = {
        allow_structured_metadata = false;
        reject_old_samples = true;
        reject_old_samples_max_age = "168h";
        max_query_series = 5000;
      };

      table_manager = {
        retention_deletes_enabled = false;
        retention_period = "0s";
      };

      compactor = {
        working_directory = "/var/lib/loki";
        compactor_ring = {
          kvstore = {
            store = "inmemory";
          };
        };
      };
    };
  };

  services.grafana = {
    enable = true;
    settings = {
      server = {
        http_port = grafanaPort;
        http_addr = "0.0.0.0";
      };
      "auth.anonymous".enabled = true;
      security.allow_embedding = true;
    };
  };

  system.stateVersion = "24.05";
}