lukegb/depot
1
0
Fork 0
Code Issues 1 Pull requests Projects Packages Activity Actions

ops/nixos: implement BFD+WG tunneling for mldn-rd

Browse source
This commit is contained in:
Luke Granger-Brown 2021-08-30 19:58:21 +01:00
parent bc1932df9b
commit 7134fe904a
13 changed files with 483 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

136
ops/nixos/blade-tuvok/default.nix
View file

@ -19,6 +19,24 @@ in {
hostName = "blade-tuvok"; hostName = "blade-tuvok";
hostId = "525229f7"; hostId = "525229f7";
firewall.allowedTCPPorts = [ 80 443 ]; firewall.allowedTCPPorts = [ 80 443 ];
firewall.allowedUDPPorts = [
# Wireguard
51820 51821
];
localCommands = ''
# Check if we already have our little minicontainer setup
ip netns list | grep -q wg-endpoint || (
ip netns add wg-endpoint
ip link add ve-wg-endpoint type veth peer name ve-int netns wg-endpoint
ip link set dev ve-wg-endpoint master br-public
ip link set dev ve-wg-endpoint up
ip -n wg-endpoint link set dev ve-int up
ip -n wg-endpoint addr add 92.118.28.252/24 dev ve-int
ip -n wg-endpoint addr add 2a09:a441::f00f/48 dev ve-int
ip -n wg-endpoint route add default via 92.118.28.1
ip -n wg-endpoint route add default via 2a09:a441::1
)
'';
}; };
my.ip.tailscale = "100.119.123.33"; my.ip.tailscale = "100.119.123.33";
my.blade.bay = 6; my.blade.bay = 6;
@ -88,4 +106,122 @@ in {
vrrp.priority = 100; vrrp.priority = 100;
}; };
networking.wireguard = let
ifBase = {
listenPort = null;
allowedIPsAsRoutes = false;
socketNamespace = "wg-endpoint";
};
peerBase.allowedIPs = [
"0.0.0.0/0"
"::/0"
];
swannBase = ifBase // {
privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey;
peers = [(peerBase // {
endpoint = null; # dynamic
publicKey = secrets.wireguard.tuvok-swann.swann.publicKey;
})];
};
in {
enable = true;
interfaces.wg-swann-vm = swannBase // {
ips = [
"2a09:a442::1:2/112"
"92.118.30.1/31"
];
listenPort = 51820;
};
interfaces.wg-swann-ee = swannBase // {
ips = [
"2a09:a442::2:2/112"
"92.118.30.3/31"
];
listenPort = 51821;
};
};
environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile {
name = "bird-wg-endpoint.conf";
text = ''
router id 92.118.28.252;
protocol kernel {
persist;
ipv4 {
import none;
export all;
};
};
protocol kernel {
persist;
ipv6 {
import none;
export all;
};
};
protocol device {};
protocol static export4 {
ipv4 {};
route 92.118.30.0/24
via 92.118.30.0 weight 1 bfd # Virgin Media
via 92.118.30.2 weight 2 bfd; # EE
};
protocol static export6 {
ipv6 {};
route 2a09:a443::/32
via 2a09:a442::1:1 weight 1 bfd # Virgin Media
via 2a09:a442::2:1 weight 2 bfd; # EE
};
protocol bfd {
interface "*" {
min rx interval 10ms;
min tx interval 50ms;
idle tx interval 1s;
multiplier 20;
};
neighbor 92.118.30.0;
neighbor 2a09:a442::1:1;
neighbor 92.118.30.2;
neighbor 2a09:a442::2:1;
};
'';
checkPhase = ''
${pkgs.bird2}/bin/bird -d -p -c $out
'';
};
systemd.services.bird-wg-endpoint = {
wantedBy = [ "multi-user.target" ];
reloadIfChanged = true;
description = "BIRD inside wg-endpoint netns";
after = [ "network.target" ];
restartTriggers = [ config.environment.etc."bird/bird-wg-endpoint.conf".source ];
serviceConfig = {
Type = "forking";
Restart = "on-failure";
CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID"
# see bird/sysdep/linux/syspriv.h
"CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ];
ProtectSystem = "full";
ProtectHome = "yes";
SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io";
MemoryDenyWriteExecute = "yes";
ExecStop = "${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl down";
ExecStart = "${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -u bird2 -g bird2 -s /var/run/bird-wg-endpoint.ctl";
ExecReload = "/bin/sh -c '${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -p && ${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl configure'";
NetworkNamespacePath = "/var/run/netns/wg-endpoint";
};
};
services.lukegbgp.config.export = {
v4Extra = ''
route 92.118.30.0/24 via 92.118.28.252;
'';
v6Extra = ''
route 2a09:a443::/32 via 2a09:a441::f00f;
'';
};
} }

18
ops/nixos/lib/bgp.nix
View file

@ -177,13 +177,25 @@ in {
type = listOf str; type = listOf str;
default = ["92.118.31.0/24"]; default = ["92.118.31.0/24"];
}; };
v4Extra = mkOption { #lukegbgp.config.export.v4Extra
type = lines;
default = "";
};
v6 = mkOption { # lukegbgp.config.export.v6 v6 = mkOption { # lukegbgp.config.export.v6
type = listOf str; type = listOf str;
default = ["2a09:a440::/48"]; default = ["2a09:a440::/48"];
}; };
v6Extra = mkOption { #lukegbgp.config.export.v6Extra
type = lines;
default = "";
};
}; };
}; };
}; };
bfd = mkOption { # lukegbgp.config.bfd
type = lines;
default = "";
};
}; };
}; };
}; };
@ -287,6 +299,7 @@ in {
}; };
}; };
${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.export.v4} ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.export.v4}
${config.services.lukegbgp.config.export.v4Extra}
}; };
protocol static export6 { protocol static export6 {
ipv6 { ipv6 {
@ -315,6 +328,11 @@ in {
}; };
}; };
${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.export.v6} ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.export.v6}
${config.services.lukegbgp.config.export.v6Extra}
};
protocol bfd {
${config.services.lukegbgp.config.bfd}
}; };
''; '';
}; };

2
ops/nixos/lib/blade-router.nix
View file

@ -280,7 +280,7 @@ in
} }
''; '';
}; };
services.radvd = { services.radvd = {
enable = true; enable = true;
config = '' config = ''

24
ops/nixos/lib/blade.nix
View file

@ -44,7 +44,7 @@ in {
]; ];
my.rundeck.tags = [ "blade" ]; my.rundeck.tags = [ "blade" ];
fileSystems = let fileSystems = let
zfs = device: { zfs = device: {
device = device; device = device;
@ -73,10 +73,10 @@ in {
fsType = "xfs"; fsType = "xfs";
}; };
}); });
boot.loader.grub.enable = true; boot.loader.grub.enable = true;
boot.loader.grub.version = 2; boot.loader.grub.version = 2;
# Networking! # Networking!
networking = { networking = {
domain = "blade.as205479.net"; domain = "blade.as205479.net";
@ -105,9 +105,9 @@ in {
address = "10.100.2.${toString (100 + config.my.blade.bay)}"; address = "10.100.2.${toString (100 + config.my.blade.bay)}";
prefixLength = 24; prefixLength = 24;
}]; }];
defaultGateway = lib.mkDefault "10.100.0.1"; defaultGateway = lib.mkDefault "10.100.0.1";
firewall.allowedUDPPorts = [ firewall.allowedUDPPorts = [
41641 # Tailscale 41641 # Tailscale
]; ];
@ -128,14 +128,14 @@ in {
'') + (lib.optionalString (config.my.blade.macAddress.internet != null) '' '') + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet" ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
''); '');
virtualisation.podman.enable = true; virtualisation.podman.enable = true;
environment.systemPackages = with pkgs; [ environment.systemPackages = with pkgs; [
ceph ceph
xfsprogs xfsprogs
]; ];
services.ceph = { services.ceph = {
enable = true; enable = true;
global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c"; global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
@ -158,15 +158,15 @@ in {
description = "Ceph OSD pre-start"; description = "Ceph OSD pre-start";
before = [ "network-online.target" "ceph-osd.target" ]; before = [ "network-online.target" "ceph-osd.target" ];
wantedBy = [ "ceph-osd.target" ]; wantedBy = [ "ceph-osd.target" ];
path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ]; path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];
serviceConfig = { serviceConfig = {
Type = "oneshot"; Type = "oneshot";
ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd"; ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
}; };
}; };
virtualisation.libvirtd = { virtualisation.libvirtd = {
enable = true; enable = true;
qemuRunAsRoot = false; qemuRunAsRoot = false;
@ -187,7 +187,7 @@ in {
Storage=none Storage=none
ProcessSizeMax=0 ProcessSizeMax=0
''; '';
system.stateVersion = "21.05"; system.stateVersion = "21.05";
}; };
} }

10
ops/nixos/lib/bvm.nix
View file

@ -27,7 +27,7 @@
]; ];
powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; powerManagement.cpuFreqGovernor = lib.mkDefault "performance";
fileSystems = { fileSystems = {
"/" = { "/" = {
device = "/dev/vda1"; device = "/dev/vda1";
@ -38,7 +38,7 @@
fsType = "vfat"; fsType = "vfat";
}; };
}; };
# Use the systemd-boot EFI boot loader. # Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true; boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true; boot.loader.efi.canTouchEfiVariables = true;
@ -46,7 +46,7 @@
nix.maxJobs = lib.mkDefault 2; nix.maxJobs = lib.mkDefault 2;
my.rundeck.tags = [ "bvm" ]; my.rundeck.tags = [ "bvm" ];
# Networking! # Networking!
networking = { networking = {
domain = "blade.as205479.net"; domain = "blade.as205479.net";
@ -60,12 +60,12 @@
address = "10.100.0.1"; address = "10.100.0.1";
interface = "enp1s0"; interface = "enp1s0";
}; };
firewall.allowedUDPPorts = [ firewall.allowedUDPPorts = [
41641 # Tailscale 41641 # Tailscale
]; ];
}; };
services.qemuGuest.enable = true; services.qemuGuest.enable = true;
}; };
} }

7
ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa
View file

@ -3,6 +3,11 @@
; SPDX-License-Identifier: Apache-2.0 ; SPDX-License-Identifier: Apache-2.0
; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL
@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 1 600 450 3600 300 @ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300
$INCLUDE tmpl.ns $INCLUDE tmpl.ns
1.0.0.0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.vm-tuvok.mldn-rd.as205479.net.
2.0.0.0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net.
1.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net.
2.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net.

4
ops/nixos/lib/coredns/zones/db.28.118.92.in-addr.arpa
View file

@ -3,7 +3,7 @@
; SPDX-License-Identifier: Apache-2.0 ; SPDX-License-Identifier: Apache-2.0
; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL
@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 12 600 450 3600 300 @ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 13 600 450 3600 300
$INCLUDE tmpl.ns $INCLUDE tmpl.ns
@ -259,7 +259,7 @@ $INCLUDE tmpl.ns
249 600 IN PTR 92-118-28-249.ptr.as205479.net. 249 600 IN PTR 92-118-28-249.ptr.as205479.net.
250 600 IN PTR 92-118-28-250.ptr.as205479.net. 250 600 IN PTR 92-118-28-250.ptr.as205479.net.
251 600 IN PTR 92-118-28-251.ptr.as205479.net. 251 600 IN PTR 92-118-28-251.ptr.as205479.net.
252 600 IN PTR 92-118-28-252.ptr.as205479.net. 252 600 IN PTR wg-gw.public.as205479.net.
253 600 IN PTR blade-paris.public.as205479.net. 253 600 IN PTR blade-paris.public.as205479.net.
254 600 IN PTR blade-tuvok.public.as205479.net. 254 600 IN PTR blade-tuvok.public.as205479.net.
255 600 IN PTR 92-118-28-255.ptr.as205479.net. 255 600 IN PTR 92-118-28-255.ptr.as205479.net.

4
ops/nixos/lib/coredns/zones/db.3.4.4.a.9.0.a.2.ip6.arpa
View file

@ -3,6 +3,8 @@
; SPDX-License-Identifier: Apache-2.0 ; SPDX-License-Identifier: Apache-2.0
; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL
@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 1 600 450 3600 300 @ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300
$INCLUDE tmpl.ns $INCLUDE tmpl.ns
1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn-rd.as205479.net.

12
ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa
View file

@ -3,14 +3,14 @@
; SPDX-License-Identifier: Apache-2.0 ; SPDX-License-Identifier: Apache-2.0
; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL
@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 1 600 450 3600 300 @ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300
$INCLUDE tmpl.ns $INCLUDE tmpl.ns
0 600 IN PTR 92-118-30-0.ptr.as205479.net. 0 600 IN PTR mldn.vm-tuvok.mldn-rd.as205479.net.
1 600 IN PTR 92-118-30-1.ptr.as205479.net. 1 600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net.
2 600 IN PTR 92-118-30-2.ptr.as205479.net. 2 600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net.
3 600 IN PTR 92-118-30-3.ptr.as205479.net. 3 600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net.
4 600 IN PTR 92-118-30-4.ptr.as205479.net. 4 600 IN PTR 92-118-30-4.ptr.as205479.net.
5 600 IN PTR 92-118-30-5.ptr.as205479.net. 5 600 IN PTR 92-118-30-5.ptr.as205479.net.
6 600 IN PTR 92-118-30-6.ptr.as205479.net. 6 600 IN PTR 92-118-30-6.ptr.as205479.net.
@ -261,5 +261,5 @@ $INCLUDE tmpl.ns
251 600 IN PTR 92-118-30-251.ptr.as205479.net. 251 600 IN PTR 92-118-30-251.ptr.as205479.net.
252 600 IN PTR 92-118-30-252.ptr.as205479.net. 252 600 IN PTR 92-118-30-252.ptr.as205479.net.
253 600 IN PTR 92-118-30-253.ptr.as205479.net. 253 600 IN PTR 92-118-30-253.ptr.as205479.net.
254 600 IN PTR 92-118-30-254.ptr.as205479.net. 254 600 IN PTR mldn-rd.as205479.net.
255 600 IN PTR 92-118-30-255.ptr.as205479.net. 255 600 IN PTR 92-118-30-255.ptr.as205479.net.

16
ops/nixos/lib/coredns/zones/db.as205479.net
View file

@ -3,7 +3,7 @@
; SPDX-License-Identifier: Apache-2.0 ; SPDX-License-Identifier: Apache-2.0
; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL
@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 26 600 450 3600 300 @ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 27 600 450 3600 300
; NB: this are also glue records in Google Domains. ; NB: this are also glue records in Google Domains.
$INCLUDE tmpl.ns $INCLUDE tmpl.ns
@ -75,6 +75,18 @@ fp-la.int 3600 IN A 208.77.235.186
fp-la-sec.int 3600 IN A 206.214.42.86 fp-la-sec.int 3600 IN A 206.214.42.86
fp-la.int 3600 IN A 206.214.42.86 fp-la.int 3600 IN A 206.214.42.86
; mldn rd
mldn.vm-tuvok.mldn-rd 3600 IN A 92.118.30.0
mldn.vm-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::1:1
tuvok.vm-tuvok.mldn-rd 3600 IN A 92.118.30.1
tuvok.vm-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::1:2
mldn.ee-tuvok.mldn-rd 3600 IN A 92.118.30.2
mldn.ee-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::2:1
tuvok.ee-tuvok.mldn-rd 3600 IN A 92.118.30.3
tuvok.ee-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::2:2
mldn-rd 3600 IN A 92.118.30.254
mldn-rd 3600 IN AAAA 2a09:a443::1
; blade internal ; blade internal
blade-oa.blade 3600 IN A 10.100.1.200 blade-oa.blade 3600 IN A 10.100.1.200
blade-vcenet1.blade 3600 IN A 10.100.1.201 blade-vcenet1.blade 3600 IN A 10.100.1.201
@ -125,6 +137,8 @@ _ceph-mon._tcp.storage.blade 60 IN SRV 10 10 6789 blade-paris.storage.blade.a
; public ; public
gw.public 3600 IN A 92.118.28.1 gw.public 3600 IN A 92.118.28.1
gw.public 3600 IN AAAA 2a09:a441::1 gw.public 3600 IN AAAA 2a09:a441::1
wg-gw.public 3600 IN A 92.118.28.252
wg-gw.public 3600 IN AAAA 2a09:a441::f00f
blade-tuvok.public 3600 IN A 92.118.28.254 blade-tuvok.public 3600 IN A 92.118.28.254
blade-tuvok.public 3600 IN AAAA 2a09:a441::ffff blade-tuvok.public 3600 IN AAAA 2a09:a441::ffff
blade-paris.public 3600 IN A 92.118.28.253 blade-paris.public 3600 IN A 92.118.28.253

2
ops/nixos/lib/switch-prebuilt.nix
View file

@ -13,7 +13,7 @@ pkgs.writeShellScriptBin "switch-prebuilt" ''
if [[ "$system" == "latest" ]]; then if [[ "$system" == "latest" ]]; then
tmpdir="$(mktemp -d)" tmpdir="$(mktemp -d)"
trap '{ rm -rf -- "$tmpdir"; }' EXIT trap '{ rm -rf -- "$tmpdir"; }' EXIT
${pkgs.curl}/bin/curl -so "$tmpdir/archive.zip" 'https://hg.lukegb.com/api/v4/projects/lukegb%2Fdepot/jobs/artifacts/branch%2Fdefault/download?job=nixCache' ${pkgs.curl}/bin/curl -so "$tmpdir/archive.zip" 'https://hg.lukegb.com/api/v4/projects/lukegb%2Fdepot/jobs/artifacts/branch%2Fdefault/download?job=nixCache'
${pkgs.unzip}/bin/unzip -d "$tmpdir" -q -o "$tmpdir/archive.zip" ${pkgs.unzip}/bin/unzip -d "$tmpdir" -q -o "$tmpdir/archive.zip"
system="$(${pkgs.jq}/bin/jq -r ".\"$(hostname)\"" "$tmpdir/systems.json")" system="$(${pkgs.jq}/bin/jq -r ".\"$(hostname)\"" "$tmpdir/systems.json")"

4
ops/nixos/swann/README.md
View file

@ -14,5 +14,5 @@ Hardware running NixOS in my flat.
NICs on: NICs on:
* `ens-virginmedia` Virgin Media (DHCP) * `en-virginmedia` Virgin Media (DHCP)
* `ens-general` General (192.168.1.1) * `en-general` General (192.168.1.1)

343
ops/nixos/swann/default.nix
View file

@ -6,6 +6,11 @@
let let
inherit (depot.ops) secrets; inherit (depot.ops) secrets;
in { in {
imports = [
# We include this just so it sets some sysctls and firewall settings.
../lib/bgp.nix
];
boot.initrd.availableKernelModules = [ boot.initrd.availableKernelModules = [
"sd_mod" "sd_mod"
"ahci" "ahci"
@ -33,57 +38,144 @@ in {
# Networking! # Networking!
networking = { networking = {
# Routing tables:
# bgp (150) -- contains default routes over WG tunnels
# ee (201) -- table contains a static default route via EE
# main (254) -- DHCP routes (aka VM)
# Conventional lookup order is bgp, main. ee has routing rules.
hostName = "swann"; # Define your hostname. hostName = "swann"; # Define your hostname.
domain = "int.as205479.net"; domain = "int.as205479.net";
nameservers = ["8.8.8.8" "8.8.4.4"]; nameservers = ["8.8.8.8" "8.8.4.4"];
useDHCP = false; useDHCP = false;
interfaces = { interfaces = {
ens-virginmedia = { lo = {
ipv4.addresses = [
{ address = "127.0.0.1"; prefixLength = 8; }
{ address = "92.118.30.254"; prefixLength = 32; }
];
};
en-virginmedia = {
useDHCP = true; useDHCP = true;
macAddress = "e4:3a:6e:16:07:61"; macAddress = "e4:3a:6e:16:07:61";
}; };
ens-general = { en-ee = {
ipv4.addresses = [
{ address = "192.168.200.2"; prefixLength = 24; }
];
ipv4.routes = [{
via = "192.168.200.1";
address = "0.0.0.0";
prefixLength = 0;
options.table = "201";
}];
};
en-general = {
ipv4.addresses = [ ipv4.addresses = [
{ address = "192.168.1.1"; prefixLength = 23; } { address = "192.168.1.1"; prefixLength = 23; }
]; ];
ipv6.addresses = [ ipv6.addresses = [
{ address = "2a02:88fd:f:d::2"; prefixLength = 64; } { address = "2a09:a443::1"; prefixLength = 64; }
]; { address = "2a09:a443:1::1"; prefixLength = 48; }
ipv6.routes = [
{ address = "2a02:88fd:f:d::"; prefixLength = 64; options.metric = "100"; }
]; ];
}; };
}; };
dhcpcd.extraConfig = ''
interface en-virginmedia
metric 100
interface en-ee
metric 250
'';
localCommands = let
claimedPriorities = { min = 10000; max = 10100; };
rules = [
# Route traffic to EE via WG... via EE.
{ priority = 10000; both = "fwmark 0xdead table 201"; }
# Route traffic to VM via WG... via VM DHCP in main table.
{ priority = 10001; both = "fwmark 0xbeef table main"; }
# Make ping work over the tunnels.
{ priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
{ priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
# Now some subset of RFC1918 via main table too.
{ priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
{ priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
{ priority = 10022; v4 = "to 172.16.0.0/12 table main"; }
# And the linknets...
{ priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
{ priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
{ priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
# Everything else over WG.
{ priority = 10030; both = "table 150"; }
];
clearRules = map (x: ''
ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
'') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
assert (both == "" || (v4 == "" && v6 == ""));
assert priority >= claimedPriorities.min;
assert priority < claimedPriorities.max;
let
rv4 = if v4 != "" then v4 else both;
rv6 = if v6 != "" then v6 else both;
in ''
${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
'';
addRules = map ruleToLine rules;
in ''
${lib.concatStringsSep "\n" clearRules}
${lib.concatStringsSep "\n" addRules}
ip -4 route flush table 151 >/dev/null 2>&1 || true
ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
ip -6 route flush table 151 >/dev/null 2>&1 || true
ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151
ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151
ip -4 route flush table 152 >/dev/null 2>&1 || true
ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151
ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152
ip -6 route flush table 152 >/dev/null 2>&1 || true
ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152
ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152
'';
}; };
my.ip.tailscale = "100.102.224.95"; my.ip.tailscale = "100.102.224.95";
services.udev.extraRules = '' services.udev.extraRules = ''
ATTR{address}=="e4:3a:6e:16:07:62", NAME="ens-virginmedia" ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia"
ATTR{address}=="e4:3a:6e:16:07:67", NAME="ens-general" ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee"
ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general"
''; '';
boot.kernel.sysctl = { boot.kernel.sysctl = {
"net.ipv4.ip_forward" = "1"; "net.ipv4.ip_forward" = "1";
"net.ipv6.conf.default.forwarding" = "1"; "net.ipv6.conf.default.forwarding" = "1";
"net.ipv6.conf.all.forwarding" = "1"; "net.ipv6.conf.all.forwarding" = "1";
"net.ipv6.conf.ens-virginmedia.accept_ra" = "2"; "net.ipv6.conf.en-virginmedia.accept_ra" = "2";
}; };
networking.nat = { networking.nat = {
enable = true; enable = true;
externalInterface = "ens-virginmedia"; internalInterfaces = ["en-general"];
internalInterfaces = ["ens-general"]; externalInterface = "en-virginmedia";
forwardPorts = [ extraCommands = ''
{ destination = "192.168.1.40:22"; proto = "tcp"; sourcePort = 10022; } # NAT packets going over EE plain.
{ destination = "192.168.1.40:41641"; proto = "udp"; sourcePort = 41641; } iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
{ destination = "192.168.1.40:80"; proto = "tcp"; sourcePort = 80; }
{ destination = "192.168.1.40:443"; proto = "tcp"; sourcePort = 443; }
# IPFS # SNAT packets we're sending over tunnels.
{ destination = "192.168.1.40:4001"; proto = "tcp"; sourcePort = 4001; } iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254
{ destination = "192.168.1.40:4001"; proto = "udp"; sourcePort = 4001; } iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
]; '';
}; };
services.dhcpd4 = { services.dhcpd4 = {
enable = true; enable = true;
interfaces = ["ens-general"]; interfaces = ["en-general"];
authoritative = true; authoritative = true;
extraConfig = '' extraConfig = ''
subnet 192.168.1.0 netmask 255.255.255.0 { subnet 192.168.1.0 netmask 255.255.255.0 {
@ -93,6 +185,7 @@ in {
option domain-name "house.as205479.net"; option domain-name "house.as205479.net";
default-lease-time 600; default-lease-time 600;
max-lease-time 3600; max-lease-time 3600;
option interface-mtu 1420; # Wireguard
range 192.168.1.100 192.168.1.200; range 192.168.1.100 192.168.1.200;
} }
@ -120,18 +213,50 @@ in {
} }
]; ];
}; };
networking.localCommands = '' networking.wireguard = let
tc qdisc del dev ens-virginmedia root || true ifBase = {
tc qdisc add dev ens-virginmedia root cake bandwidth 20Mbit docsis nat dual-srchost listenPort = null;
allowedIPsAsRoutes = false;
ip link add name ifb-virginmedia type ifb || true };
tc qdisc del dev ens-virginmedia ingress || true peerBase = {
tc qdisc add dev ens-virginmedia handle ffff: ingress allowedIPs = [
tc qdisc del dev ifb-virginmedia root || true "0.0.0.0/0"
tc qdisc add dev ifb-virginmedia root cake bandwidth 450Mbit besteffort docsis nat wash dual-dsthost "::/0"
ip link set dev ifb-virginmedia up ];
tc filter add dev ens-virginmedia parent ffff: matchall action mirred egress redirect dev ifb-virginmedia };
''; in {
enable = true;
interfaces.wg-tuvok-vm = ifBase // {
ips = [
"2a09:a442::1:1/112"
"92.118.30.0/31"
];
listenPort = 51820;
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
peers = [(peerBase // {
endpoint = "92.118.28.252:51820";
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
})];
postSetup = ''
wg set wg-tuvok-vm fwmark 0xbeef
'';
};
interfaces.wg-tuvok-ee = ifBase // {
ips = [
"2a09:a442::2:1/112"
"92.118.30.2/31"
];
listenPort = 51821;
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
peers = [(peerBase // {
endpoint = "92.118.28.252:51821";
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
})];
postSetup = ''
wg set wg-tuvok-ee fwmark 0xdead
'';
};
};
services.unifi = { services.unifi = {
enable = true; enable = true;
@ -149,7 +274,7 @@ in {
}; };
networking.firewall = { networking.firewall = {
interfaces.ens-general = { interfaces.en-general = {
allowedTCPPorts = [ allowedTCPPorts = [
8080 6789 # Unifi 8080 6789 # Unifi
53 # DNS 53 # DNS
@ -159,6 +284,20 @@ in {
53 # DNS 53 # DNS
]; ];
}; };
interfaces.wg-tuvok-ee = {
allowedUDPPorts = [
3784 # BFD
];
};
interfaces.wg-tuvok-vm = {
allowedUDPPorts = [
3784 # BFD
];
};
extraCommands = ''
iptables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip6tables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
'';
}; };
services.ddclient = { services.ddclient = {
@ -169,7 +308,7 @@ in {
password = secrets.cloudflareCredentials.token; password = secrets.cloudflareCredentials.token;
use = "if"; use = "if";
extraConfig = '' extraConfig = ''
if=ens-virginmedia if=en-virginmedia
daemon=0 daemon=0
''; '';
}; };
@ -200,15 +339,17 @@ in {
RuntimeDirectory = "ddclient"; RuntimeDirectory = "ddclient";
in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf"; in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf";
environment.systemPackages = with pkgs; []; environment.systemPackages = with pkgs; [
ethtool
];
services.coredns = { services.coredns = {
enable = true; enable = true;
config = '' config = ''
.:53 { .:53 {
bind 192.168.1.1 127.0.0.53 bind 192.168.1.1 127.0.0.53 2a09:a443::1 2a09:a443:1::1
acl { acl {
allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32
block block
} }
hosts /dev/null { hosts /dev/null {
@ -261,38 +402,106 @@ in {
]; ];
}; };
# This is cursed. services.bird2 = {
services.ndppd = { enable = true;
enable = false;
proxies.ens-virginmedia = {
router = false;
rules."2a02:88fd:f:d::/64" = {
method = "iface";
interface = "ens-general";
};
};
};
networking.dhcpcd.extraConfig = ''
noipv6rs
interface ens-virginmedia
ipv6rs
iaid 1
ia_na 2
ia_pd 3 ens-general/1/64
'';
services.radvd = {
enable = false;
config = '' config = ''
interface ens-general { router id 92.118.30.254;
AdvSendAdvert on;
MaxRtrAdvInterval 60; protocol kernel {
AdvDefaultLifetime 180; kernel table 150;
prefix 2a02:88fd:f:d::/64 { metric 0;
AdvValidLifetime 7200; ipv4 {
AdvPreferredLifetime 3600; import none;
export all;
}; };
}; };
protocol kernel {
kernel table 150;
metric 0;
ipv6 {
import none;
export all;
};
};
protocol device {};
protocol static export4 {
ipv4 {};
route 0.0.0.0/0 via 92.118.30.1 bfd {
# Virgin Media
preference = 100;
};
route 0.0.0.0/0 via 92.118.30.3 bfd {
# EE
preference = 10;
};
};
protocol static export6 {
ipv6 {};
route ::/0 via 2a09:a442::1:2 bfd {
# Virgin Media
preference = 100;
};
route ::/0 via 2a09:a442::2:2 bfd {
# EE
preference = 10;
};
# Covering route...
route 2a09:a443::/64 via "en-general";
route 2a09:a443::/32 unreachable;
};
protocol bfd {
interface "*" {
min rx interval 10ms;
min tx interval 50ms;
idle tx interval 1s;
multiplier 20;
};
neighbor 92.118.30.1;
neighbor 2a09:a442::1:2;
neighbor 92.118.30.3;
neighbor 2a09:a442::2:2;
};
'';
};
services.radvd = {
enable = true;
config = ''
interface en-general {
AdvSendAdvert on;
AdvLinkMTU 1420; # Wireguard
AdvManagedFlag on;
RDNSS 2a09:a443::1 {};
DNSSL house.as205479.net {};
prefix 2a09:a443::/64 {
AdvOnLink on;
AdvAutonomous on;
};
prefix 2a09:a443:1::/48 {
AdvOnLink on;
AdvAutonomous off;
};
};
'';
};
services.dhcpd6 = {
enable = true;
interfaces = ["en-general"];
authoritative = true;
extraConfig = ''
subnet6 2a09:a443:1::/48 {
range6 2a09:a443:1:1::/64;
range6 2a09:a443:1:2::/64 temporary;
prefix6 2a09:a443:1:1000:: 2a09:a443:1:ffff:: /56;
option dhcp6.name-servers 2a09:a443:1::1;
option dhcp6.domain-search "house.as205479.net";
}
''; '';
}; };