depot/third_party/nixpkgs/nixos/tests/prometheus/federation.nix

import ../make-test-python.nix (
  { lib, pkgs, ... }:

  {
    name = "prometheus-federation";

    nodes = {
      global1 =
        { config, pkgs, ... }:
        {
          environment.systemPackages = [ pkgs.jq ];

          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

          services.prometheus = {
            enable = true;
            globalConfig.scrape_interval = "2s";

            scrapeConfigs = [
              {
                job_name = "federate";
                honor_labels = true;
                metrics_path = "/federate";

                params = {
                  "match[]" = [
                    "{job=\"node\"}"
                    "{job=\"prometheus\"}"
                  ];
                };

                static_configs = [
                  {
                    targets = [
                      "prometheus1:${toString config.services.prometheus.port}"
                      "prometheus2:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
              {
                job_name = "prometheus";
                static_configs = [
                  {
                    targets = [
                      "global1:${toString config.services.prometheus.port}"
                      "global2:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
            ];
          };
        };

      global2 =
        { config, pkgs, ... }:
        {
          environment.systemPackages = [ pkgs.jq ];

          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

          services.prometheus = {
            enable = true;
            globalConfig.scrape_interval = "2s";

            scrapeConfigs = [
              {
                job_name = "federate";
                honor_labels = true;
                metrics_path = "/federate";

                params = {
                  "match[]" = [
                    "{job=\"node\"}"
                    "{job=\"prometheus\"}"
                  ];
                };

                static_configs = [
                  {
                    targets = [
                      "prometheus1:${toString config.services.prometheus.port}"
                      "prometheus2:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
              {
                job_name = "prometheus";
                static_configs = [
                  {
                    targets = [
                      "global1:${toString config.services.prometheus.port}"
                      "global2:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
            ];
          };
        };

      prometheus1 =
        { config, pkgs, ... }:
        {
          environment.systemPackages = [ pkgs.jq ];

          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

          services.prometheus = {
            enable = true;
            globalConfig.scrape_interval = "2s";

            scrapeConfigs = [
              {
                job_name = "node";
                static_configs = [
                  {
                    targets = [
                      "node1:${toString config.services.prometheus.exporters.node.port}"
                    ];
                  }
                ];
              }
              {
                job_name = "prometheus";
                static_configs = [
                  {
                    targets = [
                      "prometheus1:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
            ];
          };
        };

      prometheus2 =
        { config, pkgs, ... }:
        {
          environment.systemPackages = [ pkgs.jq ];

          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

          services.prometheus = {
            enable = true;
            globalConfig.scrape_interval = "2s";

            scrapeConfigs = [
              {
                job_name = "node";
                static_configs = [
                  {
                    targets = [
                      "node2:${toString config.services.prometheus.exporters.node.port}"
                    ];
                  }
                ];
              }
              {
                job_name = "prometheus";
                static_configs = [
                  {
                    targets = [
                      "prometheus2:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
            ];
          };
        };

      node1 =
        { config, pkgs, ... }:
        {
          services.prometheus.exporters.node = {
            enable = true;
            openFirewall = true;
          };
        };

      node2 =
        { config, pkgs, ... }:
        {
          services.prometheus.exporters.node = {
            enable = true;
            openFirewall = true;
          };
        };
    };

    testScript = ''
      for machine in node1, node2:
        machine.wait_for_unit("prometheus-node-exporter")
        machine.wait_for_open_port(9100)

      for machine in prometheus1, prometheus2, global1, global2:
        machine.wait_for_unit("prometheus")
        machine.wait_for_open_port(9090)

      # Verify both servers got the same data from the exporter
      for machine in prometheus1, prometheus2:
        machine.wait_until_succeeds(
          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
        )
        machine.wait_until_succeeds(
          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
          + "jq '.data.result[0].value[1]' | grep '\"1\"'"
        )

      for machine in global1, global2:
        machine.wait_until_succeeds(
          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
          + "jq '.data.result[0].value[1]' | grep '\"2\"'"
        )

        machine.wait_until_succeeds(
          "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
          + "jq '.data.result[0].value[1]' | grep '\"4\"'"
        )
    '';
  }
)