From 1ed83bd25adbc4e38de56ade79f266ca3be4aee5 Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <hg@lukegb.com>
Date: Tue, 9 Feb 2021 01:17:54 +0000
Subject: [PATCH] ops/nixos/blade: add ceph

---
 ops/nixos/blade-chakotay/default.nix |  4 ++++
 ops/nixos/blade-janeway/default.nix  |  6 ++++++
 ops/nixos/blade-kim/default.nix      |  4 ++++
 ops/nixos/blade-paris/default.nix    |  5 +++++
 ops/nixos/blade-torres/default.nix   |  4 ++++
 ops/nixos/blade-tuvok/default.nix    |  5 +++++
 ops/nixos/lib/blade.nix              | 22 ++++++++++++++++++++++
 7 files changed, 50 insertions(+)

diff --git a/ops/nixos/blade-chakotay/default.nix b/ops/nixos/blade-chakotay/default.nix
index 7fd6bb7a4a..07a09d5ee9 100644
--- a/ops/nixos/blade-chakotay/default.nix
+++ b/ops/nixos/blade-chakotay/default.nix
@@ -22,4 +22,8 @@ in {
     }];
   };
   my.ip.tailscale = "100.121.11.7";
+
+  services.ceph = {
+    osd.enable = true;
+  };
 }
diff --git a/ops/nixos/blade-janeway/default.nix b/ops/nixos/blade-janeway/default.nix
index 5ba38d60fc..fb127b19e5 100644
--- a/ops/nixos/blade-janeway/default.nix
+++ b/ops/nixos/blade-janeway/default.nix
@@ -26,4 +26,10 @@ in {
     }];
   };
   my.ip.tailscale = "100.121.116.85";
+
+  services.ceph = {
+    mon.enable = true;
+    mds.enable = true;
+    osd.enable = true;
+  };
 }
diff --git a/ops/nixos/blade-kim/default.nix b/ops/nixos/blade-kim/default.nix
index 08c7435bb0..c8ca3a53a7 100644
--- a/ops/nixos/blade-kim/default.nix
+++ b/ops/nixos/blade-kim/default.nix
@@ -22,4 +22,8 @@ in {
     }];
   };
   my.ip.tailscale = "100.84.36.62";
+
+  services.ceph = {
+    osd.enable = true;
+  };
 }
diff --git a/ops/nixos/blade-paris/default.nix b/ops/nixos/blade-paris/default.nix
index 836747a61b..e43cc3d7e0 100644
--- a/ops/nixos/blade-paris/default.nix
+++ b/ops/nixos/blade-paris/default.nix
@@ -22,4 +22,9 @@ in {
     }];
   };
   my.ip.tailscale = "100.117.185.118";
+
+  services.ceph = {
+    mon.enable = true;
+    osd.enable = true;
+  };
 }
diff --git a/ops/nixos/blade-torres/default.nix b/ops/nixos/blade-torres/default.nix
index f173f4a333..74d4432380 100644
--- a/ops/nixos/blade-torres/default.nix
+++ b/ops/nixos/blade-torres/default.nix
@@ -22,4 +22,8 @@ in {
     }];
   };
   my.ip.tailscale = "100.92.118.36";
+
+  services.ceph = {
+    osd.enable = true;
+  };
 }
diff --git a/ops/nixos/blade-tuvok/default.nix b/ops/nixos/blade-tuvok/default.nix
index 4fab7300f1..0bf302d4c7 100644
--- a/ops/nixos/blade-tuvok/default.nix
+++ b/ops/nixos/blade-tuvok/default.nix
@@ -22,4 +22,9 @@ in {
     }];
   };
   my.ip.tailscale = "100.119.123.33";
+
+  services.ceph = {
+    mon.enable = true;
+    osd.enable = true;
+  };
 }
diff --git a/ops/nixos/lib/blade.nix b/ops/nixos/lib/blade.nix
index 07dfdeacb6..b29b08a9b5 100644
--- a/ops/nixos/lib/blade.nix
+++ b/ops/nixos/lib/blade.nix
@@ -47,9 +47,31 @@ in {
     };
 
     defaultGateway = "192.168.1.5";
+
+    firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
+    firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];
   };
 
   virtualisation.podman.enable = true;
 
+  environment.systemPackages = with pkgs; [
+    ceph
+    xfsprogs
+  ];
+
+  services.ceph = {
+    enable = true;
+    global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
+    global.monHost = "192.168.1.180, 192.168.1.181, 192.168.1.184";
+    global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
+    mon.daemons = [ config.networking.hostName ];
+    mds.daemons = [ config.networking.hostName ];
+    osd.daemons = [ config.networking.hostName ];
+    rgw.daemons = [ config.networking.hostName ];
+    mgr.daemons = [ config.networking.hostName ];
+    mgr.enable = config.services.ceph.mon.enable;
+    rgw.enable = true;
+  };
+
   system.stateVersion = "21.05";
 }