From 2b4f4d6b1607d76ba36375cc733cf8dafc9d0ddb Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Fri, 2 Oct 2020 14:21:49 +0000 Subject: [PATCH] kusakabe: revamp config as VM host --- ops/nixos/kusakabe/default.nix | 193 +++++++++++++++++++++++++++++++-- 1 file changed, 185 insertions(+), 8 deletions(-) diff --git a/ops/nixos/kusakabe/default.nix b/ops/nixos/kusakabe/default.nix index 6fe7822907..1404dbf72d 100644 --- a/ops/nixos/kusakabe/default.nix +++ b/ops/nixos/kusakabe/default.nix @@ -16,7 +16,7 @@ in { "xhci_pci" "ehci_pci" "usbhid" - "sd_mod" + "sd_mod" ]; boot.kernelModules = [ "kvm-intel" ]; @@ -59,20 +59,197 @@ in { useDHCP = false; defaultGateway = { address = "188.165.197.254"; - interface = "br-ext"; + interface = "eno1"; }; defaultGateway6 = { address = "2001:41d0:2:8eff:ff:ff:ff:ff"; - interface = "br-ext"; + interface = "eno1"; + }; + interfaces.eno1 = { + ipv4.addresses = [{ address = "188.165.197.49"; prefixLength = 24; }]; + ipv6.addresses = [{ address = "2001:41d0:2:8e31::1"; prefixLength = 128; }]; + proxyARP = true; }; interfaces.br-ext = { useDHCP = false; - ipv4.addresses = [{ address = "188.165.197.49"; prefixLength = 24; }]; - ipv6.addresses = [{ address = "2001:41d0:2:8e31::1"; prefixLength = 64; }]; - virtual = true; + ipv4.addresses = [{ address = "137.74.77.17"; prefixLength = 28; }]; + ipv6.addresses = [{ address = "2001:41d0:2:8e31::2"; prefixLength = 64; }]; + proxyARP = true; }; - bridges.br-ext.interfaces = [ "eno1" ]; - firewall.allowPing = true; + bridges.br-ext.interfaces = []; + firewall = { + interfaces.br-ext.allowedUDPPorts = [ + # DHCP + 67 68 + + # TFTP + 69 + + # NFS + 111 2049 + ]; + interfaces.br-ext.allowedTCPPorts = [ + # NFS + 111 2049 + + # k8s + 80 443 6443 22623 + ]; + allowedTCPPorts = [ 80 443 ]; + }; + }; + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1; + boot.kernel.sysctl."net.ipv6.conf.br-ext.proxy_ndp" = 1; + boot.kernel.sysctl."net.ipv6.conf.eno1.proxy_ndp" = 1; + + services.dhcpd4 = { + enable = true; + interfaces = ["br-ext"]; + authoritative = true; + extraConfig = '' + option arch code 93 = unsigned integer 16; + subnet 137.74.77.16 netmask 255.255.255.240 { + option subnet-mask 255.255.255.240; + option routers 137.74.77.17; + option domain-name-servers 8.8.8.8, 8.8.4.4; + option domain-name "kusakabe.lukegb.tech"; + deny unknown-clients; + + class "pxeclients" { + match if substring (option vendor-class-identifier, 0, 9) = "PXEClient"; + next-server 137.74.77.17; + } + class "httpclients" { + match if substring (option vendor-class-identifier, 0, 10) = "HTTPClient"; + option vendor-class-identifier "HTTPClient"; + } + if exists user-class and option user-class = "iPXE" { + filename "http://fed.lukegb.tech/ipxe.ipxe"; + } else if option arch = 00:07 { + # x86-64 UEFI + filename "netboot.xyz-snponly.efi"; + } else { + # Legacy PXE + filename "netboot.xyz-undionly.kpxe"; + } + } + ''; + machines = [{ + hostName = "fed"; + ethernetAddress = "02:00:00:d1:92:7a"; + ipAddress = "137.74.77.18"; + } { + hostName = "okdboot"; + ethernetAddress = "52:54:00:52:40:96"; + ipAddress = "137.74.77.20"; + } { + hostName = "okd1"; + ethernetAddress = "52:54:00:be:f4:2f"; + ipAddress = "137.74.77.21"; + } { + hostName = "okd2"; + ethernetAddress = "52:54:00:92:86:09"; + ipAddress = "137.74.77.22"; + } { + hostName = "okd3"; + ethernetAddress = "52:54:00:64:3d:3b"; + ipAddress = "137.74.77.23"; + }]; + }; + services.radvd = { + enable = true; + config = '' + interface br-ext { + AdvSendAdvert on; + prefix 2001:41d0:2:8e31::/64 { }; + RDNSS 2001:4860:4860::8888 2001:4860:4860::8844 { }; + }; + ''; + }; + services.ndppd = { + enable = true; + proxies.eno1.rules."2001:41d0:2:8e31::/64" = { + method = "static"; + }; + }; + services.atftpd = { + enable = true; + extraOptions = [ + "--bind-address=137.74.77.17" + ]; + }; + + # NFS + services.nfs.server = { + enable = true; + exports = '' + /export 137.74.77.16/28(rw,sync,nohide,no_subtree_check,no_root_squash,wdelay,fsid=0,insecure,crossmnt) + /export/openshift 137.74.77.16/28(rw,sync,nohide,no_subtree_check,no_root_squash,no_wdelay,insecure,crossmnt) + ''; + }; + + # LB + services.haproxy = { + enable = true; + config = '' + global + maxconn 50000 + nbthread 4 + + defaults + log global + mode tcp + option tcplog + maxconn 3000 + timeout connect 10s + timeout client 1m + timeout server 1m + + frontend k8sapi + bind 137.74.77.17:6443 + default_backend k8sapi-backend + + backend k8sapi-backend + balance roundrobin + mode tcp + server okd1 137.74.77.21:6443 check + server okd2 137.74.77.22:6443 check + server okd3 137.74.77.23:6443 check + + frontend machineconfig + bind 137.74.77.17:22623 + default_backend machineconfig-backend + + backend machineconfig-backend + balance roundrobin + mode tcp + server okd1 137.74.77.21:22623 check + server okd2 137.74.77.22:22623 check + server okd3 137.74.77.23:22623 check + + frontend https + bind 137.74.77.17:443 + default_backend https-backend + + backend https-backend + balance roundrobin + mode tcp + server okd1 137.74.77.21:443 check + server okd2 137.74.77.22:443 check + server okd3 137.74.77.23:443 check + + frontend http + bind 137.74.77.17:80 + default_backend http-backend + + backend http-backend + balance roundrobin + mode tcp + server okd1 137.74.77.21:80 check + server okd2 137.74.77.22:80 check + server okd3 137.74.77.23:80 check + ''; }; virtualisation.libvirtd = {