From fee02312d3ca241cf977a1028a3fd1ef13f6325c Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <hg@lukegb.com>
Date: Fri, 17 Dec 2021 00:27:24 +0000
Subject: [PATCH] blade-tuvok: move public interface off a VLAN

Previously, the public/internal interfaces were VLANned onto the same NIC. For
some reason, sometime the Emulex adapters seem to end up not getting configured
properly, which causes me no end of pain when I spend time trying to debug why
none of my VMs can see the internet anymore.

Instead of doing this, put the public interface onto its own actual virtual
network interface.
---
 ops/nixos/blade-tuvok/default.nix |  3 ++-
 ops/nixos/lib/blade.nix           | 20 +++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/ops/nixos/blade-tuvok/default.nix b/ops/nixos/blade-tuvok/default.nix
index 2033c84918..6f413dbc23 100644
--- a/ops/nixos/blade-tuvok/default.nix
+++ b/ops/nixos/blade-tuvok/default.nix
@@ -47,7 +47,8 @@ in {
   my.blade.macAddress = {
     internal = "e4:11:5b:ac:e3:fe";
     storage = "e4:11:5b:ac:e4:02";
-    internet = "e4:11:5b:ac:e4:00";
+    internet = "e4:11:5b:ac:e3:ff";
+    public = "e4:11:5b:ac:e4:00";
   };
 
   services.ceph = {
diff --git a/ops/nixos/lib/blade.nix b/ops/nixos/lib/blade.nix
index 2f55fbfe90..9f6525de97 100644
--- a/ops/nixos/lib/blade.nix
+++ b/ops/nixos/lib/blade.nix
@@ -24,6 +24,11 @@ in {
       type = lib.types.nullOr lib.types.str;
       default = null;
     };
+    macAddress.public = lib.mkOption {
+      # If not using a VLAN.
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+    };
   };
 
   config = {
@@ -90,12 +95,15 @@ in {
         br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
       in {
         br-mgmt = br [ "en-int" ];
-        br-public = br [ "vl-int-public" ];
-      };
-      vlans.vl-int-public = {
-        id = 100;
-        interface = "en-int";
+        br-public = br [ (if config.my.blade.macAddress.public == null then "vl-int-public" else "en-public") ];
       };
+      vlans = ({} //
+        (if config.my.blade.macAddress.public == null then {
+          vl-int-public = {
+            id = 100;
+            interface = "en-int";
+          };
+        } else {}));
 
       interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
         address = "10.100.0.${toString (100 + config.my.blade.bay)}";
@@ -132,6 +140,8 @@ in {
       ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
     '') + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
       ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
+    '') + (lib.optionalString (config.my.blade.macAddress.public != null) ''
+      ATTR{address}=="${config.my.blade.macAddress.public}", NAME="en-public"
     '');
 
     virtualisation.podman.enable = true;