ops/nixos: -blade
This commit is contained in:
parent
8ee5597278
commit
336b75676b
11 changed files with 15 additions and 1187 deletions
|
|
@ -1,32 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/blade.nix
|
||||
];
|
||||
|
||||
boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_01012b89ef8e36218414588461cda968a6fbfa5bf39e398aaeda43d9e398b05f86800000000000000000000097b1f26d001a240091558107b6a8e160-0:0";
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
hostName = "blade-chakotay";
|
||||
hostId = "40bc5a75";
|
||||
};
|
||||
my.ip.tailscale = "100.121.11.7";
|
||||
my.blade.bay = 5;
|
||||
my.blade.macAddress = {
|
||||
internal = "e4:11:5b:ac:e3:b8";
|
||||
storage = "e4:11:5b:ac:e3:bc";
|
||||
};
|
||||
my.deploy.enable = false;
|
||||
my.rundeck.expectedOnline = false;
|
||||
|
||||
services.ceph = {
|
||||
#osd.enable = true;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/blade.nix
|
||||
];
|
||||
|
||||
boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_0401c0556e165beb5c1df7b9f438e6ba98af3caa816d88e69b2bf7ec2909f6ca293a000000000000000000009c7343f0ff88671891558107c52824c1-0:0";
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
hostName = "blade-janeway";
|
||||
hostId = "3a62390f";
|
||||
};
|
||||
my.ip.tailscale = "100.121.116.85";
|
||||
my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6279:7455";
|
||||
my.blade.bay = 3;
|
||||
my.blade.macAddress = {
|
||||
internal = "e8:39:35:1f:7f:8a";
|
||||
storage = "e8:39:35:1f:7f:8e";
|
||||
};
|
||||
|
||||
services.ceph = {
|
||||
mon.enable = true;
|
||||
mds.enable = true;
|
||||
osd = {
|
||||
enable = true;
|
||||
daemons = [ "0" "1" ];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/blade.nix
|
||||
];
|
||||
|
||||
boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_01015d3cbc98a05319a5ebb2b00fed57649bb0a8b5b5e517be322f1be132d4d8d60d00000000000000000000021d5c2c0083090091558107b6a8d5cd-0:0";
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
hostName = "blade-kim";
|
||||
hostId = "1643efb6";
|
||||
};
|
||||
my.ip.tailscale = "100.84.36.62";
|
||||
my.blade.bay = 1;
|
||||
my.blade.macAddress = {
|
||||
internal = "e4:11:5b:ac:d1:7a";
|
||||
storage = "e4:11:5b:ac:d1:7e";
|
||||
};
|
||||
my.deploy.enable = false;
|
||||
my.rundeck.expectedOnline = false;
|
||||
|
||||
services.ceph = {
|
||||
#osd.enable = true;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/blade-router.nix
|
||||
../lib/blade.nix
|
||||
];
|
||||
|
||||
boot.loader.grub.device = "/dev/disk/by-id/usb-SanDisk_Ultra_4C530001320720111165-0:0";
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
hostName = "blade-paris";
|
||||
hostId = "41b2a198";
|
||||
};
|
||||
my.ip.tailscale = "100.81.131.61";
|
||||
my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6251:833d";
|
||||
my.blade.onZFS = false;
|
||||
my.blade.bay = 2;
|
||||
my.blade.macAddress = {
|
||||
internal = "e4:11:5b:ac:e4:8a";
|
||||
storage = "e4:11:5b:ac:e4:8e";
|
||||
internet = "e4:11:5b:ac:e4:8c";
|
||||
};
|
||||
my.deploy.enable = false;
|
||||
|
||||
services.ceph = {
|
||||
mon.enable = true;
|
||||
osd = {
|
||||
enable = true;
|
||||
daemons = [ "2" ];
|
||||
};
|
||||
};
|
||||
|
||||
my.blade-router = {
|
||||
addresses.linknet = {
|
||||
v4 = { local = "195.74.55.23"; remote = "195.74.55.22"; };
|
||||
v6 = {
|
||||
local = "2a03:ee40:8080:9:2::2";
|
||||
remote = "2a03:ee40:8080:9:2::1";
|
||||
};
|
||||
};
|
||||
|
||||
addresses.br-public = {
|
||||
v4.addr = "92.118.28.253";
|
||||
v6.addr = "2a09:a441::fffe";
|
||||
};
|
||||
|
||||
vrrp.priority = 50;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/blade.nix
|
||||
];
|
||||
|
||||
boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_01011cbe2d1b827aedae2d5668fecf2a2dbdab7d0ecc0416f6e7c5485ecec8f5e5870000000000000000000080ce63590095210091558107b6a8e55d-0:0";
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
hostName = "blade-torres";
|
||||
hostId = "86db1d9c";
|
||||
|
||||
# HACK: torres' secondary intf. seems to be having problems
|
||||
# temporarily remap this onto the VLANs with everything else.
|
||||
vlans.en-storage = {
|
||||
id = 101;
|
||||
interface = "en-int";
|
||||
};
|
||||
};
|
||||
my.ip.tailscale = "100.92.118.36";
|
||||
my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:625c:7624";
|
||||
my.blade.bay = 8;
|
||||
my.blade.macAddress = {
|
||||
internal = "e4:11:5b:ac:e3:cc";
|
||||
storage = null;
|
||||
#storage = "e4:11:5b:ac:e3:d0";
|
||||
};
|
||||
|
||||
services.ceph = {
|
||||
osd = {
|
||||
enable = true;
|
||||
daemons = [ "4" "5" ];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,272 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/blade-router.nix
|
||||
../lib/blade.nix
|
||||
../lib/fup.nix
|
||||
];
|
||||
|
||||
boot.loader.grub.device = "/dev/disk/by-id/usb-SanDisk_Cruzer_Blade_00023903100122082002-0:0";
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
hostName = "blade-tuvok";
|
||||
hostId = "525229f7";
|
||||
firewall.allowedTCPPorts = [ 80 443 ];
|
||||
firewall.allowedUDPPorts = [
|
||||
# Wireguard
|
||||
51820 51821 51822
|
||||
];
|
||||
localCommands = ''
|
||||
# Check if we already have our little minicontainer setup
|
||||
ip netns list | grep -q wg-endpoint || (
|
||||
ip netns add wg-endpoint
|
||||
ip link add ve-wg-endpoint type veth peer name ve-int netns wg-endpoint
|
||||
ip link set dev ve-wg-endpoint master br-public
|
||||
ip link set dev ve-wg-endpoint up
|
||||
ip -n wg-endpoint link set dev ve-int up
|
||||
ip -n wg-endpoint addr add 92.118.28.252/24 dev ve-int
|
||||
ip -n wg-endpoint addr add 2a09:a441::f00f/48 dev ve-int
|
||||
ip -n wg-endpoint route add default via 92.118.28.1
|
||||
ip -n wg-endpoint route add default via 2a09:a441::1
|
||||
ip netns exec wg-endpoint ${pkgs.procps}/bin/sysctl net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.default.forwarding=1
|
||||
)
|
||||
|
||||
# We are forcing the MAC on vl-linx to be something particular,
|
||||
# but we also need to add the corresponding LL address.
|
||||
# This is cursed.
|
||||
ip -6 addr add fe80::e611:5bff:feac:e400/64 dev vl-linx &>/dev/null || true
|
||||
'';
|
||||
nat.extraCommands = ''
|
||||
iptables -w -t nat -A nixos-nat-post -m mark --mark 0x1 -o vl-linx -j SNAT --to-source 92.118.28.1
|
||||
'';
|
||||
};
|
||||
my.ip.tailscale = "100.119.123.33";
|
||||
my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6277:7b21";
|
||||
my.blade.bay = 6;
|
||||
my.blade.macAddress = {
|
||||
internal = "e4:11:5b:ac:e3:fe";
|
||||
storage = "e4:11:5b:ac:e4:02";
|
||||
internet = "e4:11:5b:ac:e3:ff";
|
||||
public = "e4:11:5b:ac:e4:00";
|
||||
};
|
||||
|
||||
services.ceph = {
|
||||
mon.enable = true;
|
||||
osd = {
|
||||
enable = true;
|
||||
daemons = [ "3" ];
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
recommendedTlsSettings = true;
|
||||
recommendedGzipSettings = true;
|
||||
virtualHosts."objdump.zxcvbnm.ninja" = {
|
||||
default = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://localhost:7480";
|
||||
extraConfig = ''
|
||||
proxy_redirect off;
|
||||
client_max_body_size 0;
|
||||
proxy_buffering off;
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = {
|
||||
hostnames = [ "objdump.zxcvbnm.ninja" "*.objdump.zxcvbnm.ninja" ];
|
||||
nginxVirtualHosts = [ "objdump.zxcvbnm.ninja" ];
|
||||
};
|
||||
my.fup.listen = [
|
||||
"0.0.0.0" "[::]"
|
||||
];
|
||||
|
||||
my.blade-router = {
|
||||
addresses.linknet = {
|
||||
v4 = { local = "195.74.55.21"; remote = "195.74.55.20"; };
|
||||
v6 = {
|
||||
local = "2a03:ee40:8080:9:1::2";
|
||||
remote = "2a03:ee40:8080:9:1::1";
|
||||
};
|
||||
};
|
||||
|
||||
addresses.br-public = {
|
||||
v4.addr = "92.118.28.254";
|
||||
v6.addr = "2a09:a441::ffff";
|
||||
};
|
||||
|
||||
linx.enable = true;
|
||||
|
||||
vrrp.priority = 100;
|
||||
};
|
||||
|
||||
networking.wireguard = let
|
||||
ifBase = {
|
||||
listenPort = null;
|
||||
allowedIPsAsRoutes = false;
|
||||
socketNamespace = "wg-endpoint";
|
||||
};
|
||||
peerBase.allowedIPs = [
|
||||
"0.0.0.0/0"
|
||||
"::/0"
|
||||
];
|
||||
swannPeerBase = peerBase // {
|
||||
endpoint = null; # dynamic
|
||||
publicKey = secrets.wireguard.tuvok-swann.swann.publicKey;
|
||||
};
|
||||
swannBase = ifBase // {
|
||||
privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey;
|
||||
peers = [swannPeerBase];
|
||||
};
|
||||
in {
|
||||
enable = true;
|
||||
interfaces.wg-swann-ee = swannBase // {
|
||||
ips = [
|
||||
"2a09:a442::2:2/112"
|
||||
"92.118.30.3/31"
|
||||
];
|
||||
listenPort = 51821;
|
||||
};
|
||||
interfaces.wg-swann-gnet = swannBase // {
|
||||
ips = [
|
||||
"2a09:a442::3:2/112"
|
||||
"92.118.30.5/31"
|
||||
];
|
||||
listenPort = 51822;
|
||||
peers = [(swannPeerBase // {
|
||||
endpoint = "185.250.189.204:51822";
|
||||
})];
|
||||
};
|
||||
};
|
||||
environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile {
|
||||
name = "bird-wg-endpoint.conf";
|
||||
text = ''
|
||||
router id 92.118.28.252;
|
||||
protocol kernel {
|
||||
persist;
|
||||
ipv4 {
|
||||
import none;
|
||||
export all;
|
||||
};
|
||||
};
|
||||
protocol kernel {
|
||||
persist;
|
||||
ipv6 {
|
||||
import none;
|
||||
export all;
|
||||
};
|
||||
};
|
||||
protocol device {};
|
||||
|
||||
protocol static export4 {
|
||||
ipv4 {};
|
||||
route 92.118.30.0/24 via 92.118.30.2 bfd {
|
||||
# EE
|
||||
preference = 10;
|
||||
};
|
||||
route 92.118.30.0/24 via 92.118.30.4 bfd {
|
||||
# GNetwork
|
||||
preference = 200;
|
||||
};
|
||||
};
|
||||
protocol static export6 {
|
||||
ipv6 {};
|
||||
route 2a09:a443::/32 via 2a09:a442::1:1 bfd {
|
||||
# Virgin Media
|
||||
preference = 100;
|
||||
};
|
||||
route 2a09:a443::/32 via 2a09:a442::2:1 bfd {
|
||||
# EE
|
||||
preference = 10;
|
||||
};
|
||||
route 2a09:a443::/32 via 2a09:a442::3:1 bfd {
|
||||
# GNetwork
|
||||
preference = 200;
|
||||
};
|
||||
};
|
||||
|
||||
protocol bfd {
|
||||
interface "*" {
|
||||
min rx interval 10ms;
|
||||
min tx interval 50ms;
|
||||
idle tx interval 1s;
|
||||
multiplier 20;
|
||||
};
|
||||
neighbor 92.118.30.2;
|
||||
neighbor 2a09:a442::2:1;
|
||||
neighbor 92.118.30.4;
|
||||
neighbor 2a09:a442::3:1;
|
||||
};
|
||||
'';
|
||||
checkPhase = ''
|
||||
${pkgs.bird2}/bin/bird -d -p -c $out
|
||||
'';
|
||||
};
|
||||
systemd.services.bird-wg-endpoint = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
reloadIfChanged = true;
|
||||
|
||||
description = "BIRD inside wg-endpoint netns";
|
||||
after = [ "network.target" ];
|
||||
restartTriggers = [ config.environment.etc."bird/bird-wg-endpoint.conf".source ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "forking";
|
||||
Restart = "on-failure";
|
||||
CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID"
|
||||
# see bird/sysdep/linux/syspriv.h
|
||||
"CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ];
|
||||
ProtectSystem = "full";
|
||||
ProtectHome = "yes";
|
||||
SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io";
|
||||
MemoryDenyWriteExecute = "yes";
|
||||
|
||||
ExecStop = "${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl down";
|
||||
ExecStart = "${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -u bird2 -g bird2 -s /var/run/bird-wg-endpoint.ctl";
|
||||
ExecReload = "/bin/sh -c '${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -p && ${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl configure'";
|
||||
NetworkNamespacePath = "/var/run/netns/wg-endpoint";
|
||||
};
|
||||
};
|
||||
# It is at this point he realises it would've been easier if he'd used NixOS containers instead.
|
||||
systemd.services.bird-wg-endpoint-exporter = {
|
||||
after = [ "bird-wg-endpoint.service" "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
DynamicUser = true;
|
||||
User = "bird-exporter";
|
||||
Group = "bird-exporter";
|
||||
PrivateTmp = true;
|
||||
Restart = "always";
|
||||
WorkingDirectory = "/tmp";
|
||||
SupplementaryGroups = "bird2";
|
||||
ExecStart = ''
|
||||
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
|
||||
-web.listen-address 0.0.0.0:9325 \
|
||||
-bird.socket /var/run/bird-wg-endpoint.ctl \
|
||||
-bird.v2=true \
|
||||
-format.new=true
|
||||
'';
|
||||
};
|
||||
};
|
||||
my.prometheus.additionalExporterPorts.bird-wg-endpoint-exporter = 9325;
|
||||
services.lukegbgp.config.export = {
|
||||
v4Extra = ''
|
||||
route 92.118.30.0/24 via 92.118.28.252;
|
||||
'';
|
||||
v6Extra = ''
|
||||
route 2a09:a443::/32 via 2a09:a441::f00f;
|
||||
'';
|
||||
};
|
||||
systemd.services.wireguard-wg-swann-ee.after = lib.mkAfter [ "network-local-commands.service" ];
|
||||
systemd.services.wireguard-wg-swann-gnet.after = lib.mkAfter [ "network-local-commands.service" ];
|
||||
|
||||
}
|
||||
|
|
@ -26,12 +26,6 @@ let
|
|||
"swann"
|
||||
"clouvider-lon01"
|
||||
"etheroute-lon01"
|
||||
"blade-janeway"
|
||||
"blade-tuvok"
|
||||
"blade-paris"
|
||||
"blade-torres"
|
||||
"blade-chakotay"
|
||||
"blade-kim"
|
||||
"frantech-lux01"
|
||||
"frantech-nyc01"
|
||||
"frantech-las01"
|
||||
|
|
|
|||
|
|
@ -359,8 +359,8 @@ in {
|
|||
bgp_ext_community.add((ro, 205479, 5006));
|
||||
bgp_ext_community.add((ro, 205479, 5007));
|
||||
bgp_ext_community.add((ro, 205479, 5008));
|
||||
bgp_ext_community.add((ro, 205479, 5009)); # fastly from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5010)); # ovh from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server
|
||||
|
||||
# do not export to clouvider; they do... strange things.
|
||||
|
|
@ -382,19 +382,19 @@ in {
|
|||
bgp_ext_community.add((ro, 205479, 2003)); #
|
||||
bgp_ext_community.add((ro, 205479, 3000)); # clouvider from clouvider-lon01
|
||||
bgp_ext_community.add((ro, 205479, 4000)); # frantech from frantech-nyc01/veloxserv from etheroute-lon01
|
||||
bgp_ext_community.add((ro, 205479, 4001)); # veloxserv from blade-tuvok/blade-paris
|
||||
bgp_ext_community.add((ro, 205479, 4001)); # veloxserv from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 4002)); # mercury from etheroute-lon01
|
||||
bgp_ext_community.add((ro, 205479, 5000)); # linx route collector from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5001)); # linx route server from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5002)); # facebook from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5003)); # openpeering from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5004)); # freetransitnet from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5005)); # he from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5006)); # clouvider from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5007)); # google from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5008)); # cloudflare from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5009)); # fastly from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5010)); # ovh from blade-tuvok
|
||||
bgp_ext_community.add((ro, 205479, 5000)); # linx route collector from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5001)); # linx route server from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5002)); # facebook from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5003)); # openpeering from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5004)); # freetransitnet from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5005)); # he from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5006)); # clouvider from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5007)); # google from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5008)); # cloudflare from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar
|
||||
bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server
|
||||
|
||||
# do not export to clouvider; they do... strange things.
|
||||
|
|
|
|||
|
|
@ -1,417 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ lib, config, pkgs, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.my.blade-router;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
../lib/bgp.nix
|
||||
];
|
||||
|
||||
options.my.blade-router = {
|
||||
addresses.linknet.v4 = {
|
||||
local = mkOption { type = types.str; };
|
||||
remote = mkOption { type = types.str; };
|
||||
prefixLength = mkOption { type = types.int; default = 31; };
|
||||
};
|
||||
addresses.linknet.v6 = {
|
||||
local = mkOption { type = types.str; };
|
||||
remote = mkOption { type = types.str; };
|
||||
prefixLength = mkOption { type = types.int; default = 126; };
|
||||
};
|
||||
|
||||
addresses.br-public.v4 = {
|
||||
addr = mkOption { type = types.str; };
|
||||
prefixLength = mkOption { type = types.int; default = 24; };
|
||||
};
|
||||
addresses.br-public.v6 = {
|
||||
addr = mkOption { type = types.str; };
|
||||
prefixLength = mkOption { type = types.int; default = 48; };
|
||||
};
|
||||
|
||||
addresses.br-public-vip.v4 = {
|
||||
addr = mkOption { type = types.str; default = "92.118.28.1"; };
|
||||
prefixLength = mkOption { type = types.int; default = 24; };
|
||||
};
|
||||
addresses.br-public-vip.v6 = {
|
||||
addr = mkOption { type = types.str; default = "2a09:a441::1"; };
|
||||
prefixLength = mkOption { type = types.int; default = 48; };
|
||||
};
|
||||
addresses.br-public-vip.v6-ll = {
|
||||
addr = mkOption { type = types.str; default = "fe80::f00f"; };
|
||||
prefixLength = mkOption { type = types.int; default = 64; };
|
||||
};
|
||||
|
||||
addresses.br-public-radvd-prefix = {
|
||||
addr = mkOption { type = types.str; default = "2a09:a441:0:ffff::"; };
|
||||
prefixLength = mkOption { type = types.int; default = 64; };
|
||||
};
|
||||
|
||||
linx.enable = mkEnableOption "linx";
|
||||
addresses.vl-linx.v4 = {
|
||||
addr = mkOption { type = types.str; default = "195.66.224.58"; };
|
||||
prefixLength = mkOption { type = types.int; default = 21; };
|
||||
};
|
||||
addresses.vl-linx.v6 = {
|
||||
addr = mkOption { type = types.str; default = "2001:7f8:4::3:22a7:1"; };
|
||||
prefixLength = mkOption { type = types.int; default = 64; };
|
||||
};
|
||||
|
||||
vrrp.priority = mkOption { type = types.int; };
|
||||
};
|
||||
|
||||
config = {
|
||||
boot.kernel.sysctl = {
|
||||
"net.ipv4.ip_forward" = 1;
|
||||
"net.ipv6.conf.all.forwarding" = 1;
|
||||
|
||||
"net.ipv4.conf.vl-linx.arp_announce" = 1;
|
||||
"net.ipv4.conf.vl-linx.arp_ignore" = 1;
|
||||
"net.ipv4.neigh.vl-linx.base_reachable_time_ms" = 14400000;
|
||||
"net.ipv6.neigh.vl-linx.base_reachable_time_ms" = 14400000;
|
||||
};
|
||||
|
||||
networking = {
|
||||
vlans.vl-transit = {
|
||||
id = 100;
|
||||
interface = "en-internet";
|
||||
};
|
||||
vlans.vl-linx = {
|
||||
id = 200;
|
||||
interface = "en-internet";
|
||||
};
|
||||
|
||||
interfaces.br-public.ipv4.addresses = [{
|
||||
address = config.my.blade-router.addresses.br-public.v4.addr;
|
||||
prefixLength = config.my.blade-router.addresses.br-public.v4.prefixLength;
|
||||
}];
|
||||
interfaces.br-public.ipv6.addresses = [{
|
||||
address = config.my.blade-router.addresses.br-public.v6.addr;
|
||||
prefixLength = config.my.blade-router.addresses.br-public.v6.prefixLength;
|
||||
}];
|
||||
interfaces.vl-transit.ipv4.addresses = [{
|
||||
address = config.my.blade-router.addresses.linknet.v4.local;
|
||||
prefixLength = config.my.blade-router.addresses.linknet.v4.prefixLength;
|
||||
}];
|
||||
interfaces.vl-transit.ipv6.addresses = [{
|
||||
address = config.my.blade-router.addresses.linknet.v6.local;
|
||||
prefixLength = config.my.blade-router.addresses.linknet.v6.prefixLength;
|
||||
}];
|
||||
interfaces.vl-linx.macAddress = "e4:11:5b:ac:e4:00";
|
||||
interfaces.vl-linx.ipv4.addresses = [{
|
||||
address = cfg.addresses.vl-linx.v4.addr;
|
||||
prefixLength = cfg.addresses.vl-linx.v4.prefixLength;
|
||||
}];
|
||||
interfaces.vl-linx.ipv6.addresses = [{
|
||||
address = cfg.addresses.vl-linx.v6.addr;
|
||||
prefixLength = cfg.addresses.vl-linx.v6.prefixLength;
|
||||
}];
|
||||
defaultGateway = config.my.blade-router.addresses.linknet.v4.remote;
|
||||
defaultGateway6 = config.my.blade-router.addresses.linknet.v6.remote;
|
||||
firewall.extraCommands = ''
|
||||
iptables -A INPUT -p vrrp -i br-mgmt -j ACCEPT
|
||||
ip6tables -A INPUT -p vrrp -i br-mgmt -j ACCEPT
|
||||
'';
|
||||
};
|
||||
|
||||
services.lukegbgp = {
|
||||
enable = true;
|
||||
config = let
|
||||
linx = {
|
||||
local = {
|
||||
asn = 205479;
|
||||
v4 = cfg.addresses.vl-linx.v4.addr;
|
||||
v6 = cfg.addresses.vl-linx.v6.addr;
|
||||
};
|
||||
};
|
||||
in {
|
||||
local.routerID = config.my.blade-router.addresses.linknet.v4.local;
|
||||
peering.veloxserv = {
|
||||
local = {
|
||||
asn = 205479;
|
||||
v4 = config.my.blade-router.addresses.linknet.v4.local;
|
||||
v6 = config.my.blade-router.addresses.linknet.v6.local;
|
||||
};
|
||||
remote = {
|
||||
asn = 3170;
|
||||
export_community = 4001;
|
||||
bgp_local_pref = 101;
|
||||
drop_asns = [
|
||||
15169 # prefer RS to transit
|
||||
];
|
||||
routers = [{
|
||||
v4 = config.my.blade-router.addresses.linknet.v4.remote;
|
||||
v6 = config.my.blade-router.addresses.linknet.v6.remote;
|
||||
}];
|
||||
};
|
||||
};
|
||||
peering.bgptoolscollector = {
|
||||
local = {
|
||||
asn = 205479;
|
||||
v4 = config.my.blade-router.addresses.linknet.v4.local;
|
||||
v6 = config.my.blade-router.addresses.linknet.v6.local;
|
||||
};
|
||||
remote = {
|
||||
asn = 212232;
|
||||
export_community = 5000;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "185.230.223.42";
|
||||
v6 = "2a0c:2f07:9459::b1";
|
||||
multihop = 64;
|
||||
}];
|
||||
prefix_limit.v4 = 0;
|
||||
prefix_limit.v6 = 0;
|
||||
is_route_collector = true;
|
||||
};
|
||||
};
|
||||
peering.linxcollector = linx // {
|
||||
remote = {
|
||||
asn = 5459;
|
||||
export_community = 5000;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.224.254";
|
||||
v6 = "2001:7f8:4::1553:1";
|
||||
}];
|
||||
prefix_limit.v4 = 0;
|
||||
prefix_limit.v6 = 0;
|
||||
is_route_collector = false; # don't make people sad.
|
||||
};
|
||||
};
|
||||
peering.linx = linx // {
|
||||
remote = {
|
||||
asn = 8714;
|
||||
export_community = 5001;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.230";
|
||||
v6 = "2001:7f8:4::220a:1";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.231";
|
||||
v6 = "2001:7f8:4::220a:2";
|
||||
}];
|
||||
bgp_local_pref = 109;
|
||||
must_be_next_hop = false;
|
||||
};
|
||||
};
|
||||
peering.facebook = linx // {
|
||||
remote = {
|
||||
asn = 32934;
|
||||
export_community = 5002;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.69";
|
||||
v6 = "2001:7f8:4::80a6:1";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.121";
|
||||
v6 = "2001:7f8:4::80a6:2";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.227.19";
|
||||
v6 = "2001:7f8:4::80a6:5";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.226.140";
|
||||
v6 = "2001:7f8:4::80a6:3";
|
||||
}];
|
||||
bgp_local_pref = 120;
|
||||
prefix_limit.v4 = 100;
|
||||
prefix_limit.v6 = 100;
|
||||
};
|
||||
};
|
||||
peering.openpeering = linx // {
|
||||
remote = {
|
||||
asn = 20562;
|
||||
export_community = 5003;
|
||||
passive = true; # pending
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.53";
|
||||
v6 = "2001:7f8:4::5052:1";
|
||||
}];
|
||||
bgp_local_pref = 110;
|
||||
prefix_limit.v4 = 16000;
|
||||
prefix_limit.v6 = 3000;
|
||||
};
|
||||
};
|
||||
peering.freetransitnet = linx // {
|
||||
remote = {
|
||||
asn = 212895;
|
||||
export_community = 5004;
|
||||
passive = true; # pending v6
|
||||
bgp_local_pref = 100;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.105";
|
||||
v6 = "2001:7f8:4::3:3f9f:2";
|
||||
}];
|
||||
};
|
||||
};
|
||||
peering.he = linx // {
|
||||
remote = {
|
||||
asn = 6939;
|
||||
export_community = 5005;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.224.21";
|
||||
v6 = "2001:7f8:4::1b1b:1";
|
||||
}];
|
||||
bgp_local_pref = 108;
|
||||
prefix_limit.v4 = 176000;
|
||||
prefix_limit.v6 = 156000;
|
||||
};
|
||||
};
|
||||
peering.clouvider = linx // {
|
||||
remote = {
|
||||
asn = 62240;
|
||||
export_community = 5006;
|
||||
passive = true; # pending
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.227.14";
|
||||
v6 = "2001:7f8:4::f320:1";
|
||||
}];
|
||||
bgp_local_pref = 120;
|
||||
prefix_limit.v4 = 1000;
|
||||
prefix_limit.v6 = 1000;
|
||||
};
|
||||
};
|
||||
peering.google = linx // {
|
||||
remote = {
|
||||
asn = 15169;
|
||||
export_community = 5007;
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.224.125";
|
||||
v6 = "2001:7f8:4::3b41:1";
|
||||
}];
|
||||
bgp_local_pref = 120;
|
||||
prefix_limit.v4 = 15000;
|
||||
prefix_limit.v6 = 10000;
|
||||
};
|
||||
};
|
||||
peering.cloudflare = linx // {
|
||||
remote = {
|
||||
asn = 13337;
|
||||
export_community = 5008;
|
||||
passive = true; # pending
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.179";
|
||||
v6 = "2001:7f8:4::3417:1";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.227.207";
|
||||
v6 = "2001:7f8:4::3417:2";
|
||||
}];
|
||||
bgp_local_pref = 120;
|
||||
prefix_limit.v4 = 20000;
|
||||
prefix_limit.v6 = 2000;
|
||||
};
|
||||
};
|
||||
peering.fastly = linx // {
|
||||
remote = {
|
||||
asn = 54113;
|
||||
export_community = 5009;
|
||||
passive = true; # pending
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.91";
|
||||
v6 = "2001:7f8:4::d361:1";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.227.114";
|
||||
v6 = "2001:7f8:4::d361:2";
|
||||
}];
|
||||
bgp_local_pref = 120;
|
||||
prefix_limit.v4 = 250;
|
||||
prefix_limit.v6 = 250;
|
||||
};
|
||||
};
|
||||
peering.ovh = linx // {
|
||||
remote = {
|
||||
asn = 16276;
|
||||
export_community = 5010;
|
||||
passive = true; # pending
|
||||
routers = [{
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.224.220";
|
||||
v6 = "2001:7f8:4::3f94:1";
|
||||
} {
|
||||
enabled = cfg.linx.enable;
|
||||
v4 = "195.66.225.6";
|
||||
v6 = "2001:7f8:4::3f94:2";
|
||||
}];
|
||||
bgp_local_pref = 120;
|
||||
prefix_limit.v4 = 1000;
|
||||
prefix_limit.v6 = 200;
|
||||
};
|
||||
};
|
||||
export.v4 = [ "92.118.28.0/24" ];
|
||||
export.v6 = [ "2a09:a441::/32" ];
|
||||
};
|
||||
};
|
||||
|
||||
services.keepalived = let
|
||||
mgmtBase = {
|
||||
interface = "br-mgmt";
|
||||
state = "MASTER";
|
||||
priority = config.my.blade-router.vrrp.priority;
|
||||
};
|
||||
in {
|
||||
enable = true;
|
||||
vrrpInstances.mgmtGateway4 = mgmtBase // {
|
||||
virtualIps = [
|
||||
{ addr = "10.100.0.1/23"; }
|
||||
{ addr = "${config.my.blade-router.addresses.br-public-vip.v4.addr}/${toString config.my.blade-router.addresses.br-public-vip.v4.prefixLength}"; dev = "br-public"; }
|
||||
];
|
||||
virtualRouterId = 1;
|
||||
};
|
||||
vrrpInstances.mgmtGateway6 = mgmtBase // {
|
||||
virtualIps = [
|
||||
{ addr = "${config.my.blade-router.addresses.br-public-vip.v6-ll.addr}/${toString config.my.blade-router.addresses.br-public-vip.v6-ll.prefixLength}"; dev = "br-public"; }
|
||||
{ addr = "${config.my.blade-router.addresses.br-public-vip.v6.addr}/${toString config.my.blade-router.addresses.br-public-vip.v6.prefixLength}"; dev = "br-public"; }
|
||||
];
|
||||
virtualRouterId = 2;
|
||||
};
|
||||
extraGlobalDefs = ''
|
||||
enable_script_security
|
||||
script_user root
|
||||
'';
|
||||
extraConfig = ''
|
||||
vrrp_sync_group mgmtGateway {
|
||||
group {
|
||||
mgmtGateway4
|
||||
mgmtGateway6
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
services.radvd = {
|
||||
enable = true;
|
||||
config = ''
|
||||
interface br-public {
|
||||
AdvSendAdvert on;
|
||||
MinRtrAdvInterval 30;
|
||||
MaxRtrAdvInterval 100;
|
||||
AdvRASrcAddress {
|
||||
${config.my.blade-router.addresses.br-public-vip.v6-ll.addr};
|
||||
};
|
||||
prefix ${config.my.blade-router.addresses.br-public-radvd-prefix.addr}/${toString config.my.blade-router.addresses.br-public-radvd-prefix.prefixLength} {
|
||||
AdvOnLink on;
|
||||
AdvAutonomous on;
|
||||
AdvRouterAddr off;
|
||||
};
|
||||
};
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,236 +0,0 @@
|
|||
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ depot, lib, pkgs, rebuilder, config, ... }:
|
||||
let
|
||||
inherit (depot.ops) secrets;
|
||||
in {
|
||||
imports = [
|
||||
../lib/minimal.nix
|
||||
../lib/zfs.nix
|
||||
];
|
||||
|
||||
options.my.blade = {
|
||||
bay = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
};
|
||||
macAddress.internal = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
};
|
||||
macAddress.storage = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
};
|
||||
macAddress.internet = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
};
|
||||
macAddress.public = lib.mkOption {
|
||||
# If not using a VLAN.
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
onZFS = lib.mkOption { type = lib.types.bool; default = true; };
|
||||
};
|
||||
|
||||
config = {
|
||||
boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
|
||||
boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ];
|
||||
|
||||
# Enable serial console.
|
||||
boot.loader.grub.extraConfig = ''
|
||||
serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
|
||||
terminal_input console serial
|
||||
terminal_output console serial
|
||||
'';
|
||||
boot.kernelParams = [
|
||||
"console=tty1"
|
||||
"console=ttyS0,115200" # <-- /dev/console
|
||||
|
||||
"mitigations=off"
|
||||
];
|
||||
|
||||
my.rundeck.tags = [ "blade" ];
|
||||
|
||||
fileSystems = let
|
||||
zfs = device: {
|
||||
device = device;
|
||||
fsType = "zfs";
|
||||
};
|
||||
tmpfs = size: {
|
||||
device = "none";
|
||||
fsType = "tmpfs";
|
||||
options = [ "defaults" "size=${size}" "mode=755" ];
|
||||
};
|
||||
in {
|
||||
"/boot" = {
|
||||
device = "/dev/disk/by-label/boot";
|
||||
fsType = "ext4";
|
||||
};
|
||||
"/var/log" = tmpfs "2G";
|
||||
"/var/cache" = tmpfs "16G";
|
||||
} // (lib.optionalAttrs (config.my.blade.onZFS) {
|
||||
"/" = zfs "tank/local/root";
|
||||
"/tmp" = zfs "tank/local/tmp";
|
||||
"/nix" = zfs "tank/local/nix";
|
||||
"/var" = zfs "tank/safe/var";
|
||||
"/home" = zfs "tank/safe/home";
|
||||
}) // (lib.optionalAttrs (!config.my.blade.onZFS) {
|
||||
"/" = {
|
||||
device = "/dev/disk/by-label/root";
|
||||
fsType = "ext4";
|
||||
};
|
||||
}) // (lib.optionalAttrs (config.services.ceph.osd.enable || config.services.ceph.mgr.enable || config.services.ceph.mon.enable || config.services.ceph.mgr.enable) {
|
||||
"/var/lib/ceph" = {
|
||||
device = "/dev/disk/by-label/var-lib-ceph";
|
||||
fsType = "xfs";
|
||||
};
|
||||
});
|
||||
|
||||
boot.loader.grub.enable = true;
|
||||
|
||||
# Networking!
|
||||
networking = {
|
||||
domain = "blade.as205479.net";
|
||||
nameservers = ["8.8.8.8" "8.8.4.4"];
|
||||
search = lib.mkBefore [
|
||||
"blade.as205479.net"
|
||||
"storage.blade.as205479.net"
|
||||
];
|
||||
bridges = let
|
||||
br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
|
||||
in {
|
||||
br-mgmt = br [ "en-int" ];
|
||||
br-public = br [ (if config.my.blade.macAddress.public == null then "vl-int-public" else "en-public") ];
|
||||
};
|
||||
vlans = ({} //
|
||||
(if config.my.blade.macAddress.public == null then {
|
||||
vl-int-public = {
|
||||
id = 100;
|
||||
interface = "en-int";
|
||||
};
|
||||
} else {}));
|
||||
|
||||
interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
|
||||
address = "10.100.0.${toString (100 + config.my.blade.bay)}";
|
||||
prefixLength = 23;
|
||||
}];
|
||||
interfaces.en-storage.ipv4.addresses = lib.mkBefore [{
|
||||
address = "10.100.2.${toString (100 + config.my.blade.bay)}";
|
||||
prefixLength = 24;
|
||||
}];
|
||||
|
||||
defaultGateway = lib.mkDefault "10.100.0.1";
|
||||
|
||||
firewall.allowedUDPPorts = [
|
||||
41641 # Tailscale
|
||||
];
|
||||
firewall.interfaces.en-storage.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
|
||||
firewall.interfaces.en-storage.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];
|
||||
|
||||
firewall.extraCommands = ''
|
||||
iptables -A nixos-fw -i en-storage -s 10.100.2.0/23 -j ACCEPT
|
||||
iptables -A nixos-fw -i br-mgmt -s 10.100.0.0/23 -j ACCEPT
|
||||
'';
|
||||
|
||||
nat = lib.optionalAttrs (config.my.blade.macAddress.internet != null) {
|
||||
enable = true;
|
||||
internalInterfaces = [ "br-mgmt" ];
|
||||
externalInterface = "vl-transit";
|
||||
externalIP = "92.118.28.1";
|
||||
};
|
||||
};
|
||||
services.udev.extraRules = ''
|
||||
ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int"
|
||||
'' + (lib.optionalString (config.my.blade.macAddress.storage != null) ''
|
||||
ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
|
||||
'') + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
|
||||
ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
|
||||
'') + (lib.optionalString (config.my.blade.macAddress.public != null) ''
|
||||
ATTR{address}=="${config.my.blade.macAddress.public}", NAME="en-public"
|
||||
'');
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
ceph
|
||||
xfsprogs
|
||||
];
|
||||
|
||||
services.ceph = {
|
||||
enable = true;
|
||||
global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
|
||||
global.monHost = "10.100.2.103, 10.100.2.106, 10.100.2.102";
|
||||
global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
|
||||
global.publicNetwork = "10.100.2.0/24";
|
||||
global.clusterNetwork = "10.100.2.0/24";
|
||||
extraConfig.rgw_dns_name = "objdump.zxcvbnm.ninja";
|
||||
extraConfig.rgw_data_log_backing = "omap";
|
||||
extraConfig.rgw_default_data_log_backing = "omap";
|
||||
mon.daemons = [ config.networking.hostName ];
|
||||
mds.daemons = [ config.networking.hostName ];
|
||||
rgw.daemons = [ config.networking.hostName ];
|
||||
mgr.daemons = [ config.networking.hostName ];
|
||||
mgr.enable = config.services.ceph.mon.enable;
|
||||
rgw.enable = true;
|
||||
client.enable = true;
|
||||
client.extraConfig = {
|
||||
"client.libvirt" = {
|
||||
rbd_cache = "true";
|
||||
rbd_cache_policy = "writeback";
|
||||
rbd_cache_size = "2Gi";
|
||||
rbd_cache_max_dirty = "1792Mi";
|
||||
rbd_cache_target_dirty = "128Mi";
|
||||
};
|
||||
};
|
||||
};
|
||||
systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable {
|
||||
enable = true;
|
||||
description = "Ceph OSD pre-start";
|
||||
before = [ "network-online.target" "ceph-osd.target" ];
|
||||
wantedBy = [ "ceph-osd.target" ];
|
||||
|
||||
path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
|
||||
};
|
||||
};
|
||||
|
||||
virtualisation.libvirtd = {
|
||||
enable = true;
|
||||
qemu = {
|
||||
runAsRoot = true;
|
||||
package = pkgs.qemu.override {
|
||||
gtkSupport = false;
|
||||
sdlSupport = false;
|
||||
spiceSupport = true;
|
||||
cephSupport = true;
|
||||
smartcardSupport = false;
|
||||
pulseSupport = false;
|
||||
alsaSupport = false;
|
||||
libiscsiSupport = false;
|
||||
|
||||
hostCpuOnly = true;
|
||||
};
|
||||
};
|
||||
package = pkgs.libvirt.override {
|
||||
enableCeph = true;
|
||||
enableIscsi = true;
|
||||
};
|
||||
};
|
||||
security.polkit.enable = true;
|
||||
users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];
|
||||
|
||||
# Our disk is slow; don't write to it...
|
||||
services.journald.extraConfig = ''
|
||||
Storage=volatile
|
||||
'';
|
||||
systemd.coredump.extraConfig = ''
|
||||
Storage=none
|
||||
ProcessSizeMax=0
|
||||
'';
|
||||
|
||||
system.stateVersion = "21.05";
|
||||
};
|
||||
}
|
||||
|
|
@ -287,25 +287,6 @@ in {
|
|||
}];
|
||||
globalConfig.scrape_interval = "15s";
|
||||
scrapeConfigs = (builtins.attrValues depot.ops.nixos.systemExporters) ++ [{
|
||||
job_name = "blade-oa/snmp";
|
||||
metrics_path = "/snmp";
|
||||
params = {
|
||||
module = ["hpe"];
|
||||
};
|
||||
static_configs = [{
|
||||
targets = ["10.100.1.200"];
|
||||
}];
|
||||
relabel_configs = [{
|
||||
source_labels = ["__address__"];
|
||||
target_label = "__param_target";
|
||||
} {
|
||||
source_labels = ["__param_target"];
|
||||
target_label = "instance";
|
||||
} {
|
||||
target_label = "__address__";
|
||||
replacement = "totoro:${toString config.services.prometheus.exporters.snmp.port}";
|
||||
}];
|
||||
} {
|
||||
job_name = "minotar/minotarproxy";
|
||||
scheme = "https";
|
||||
static_configs = [{
|
||||
|
|
@ -343,33 +324,9 @@ in {
|
|||
rules:
|
||||
|
||||
# Blade power
|
||||
- alert: AveragePowerUsageTooHigh
|
||||
expr: (sum(avg_over_time(cpqRackPowerSupplyCurPwrOutput{job="blade-oa/snmp"}[10m])) / 230) > 6.5
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Blade: Power Usage Too High (rolling)"
|
||||
description: "Power usage of blade system has been too high for last 10 minutes ({{ $value }}). https://grafana.int.lukegb.com/d/g-u3XQ8Gk/blade-power"
|
||||
- alert: PowerUsageTooHigh
|
||||
expr: (sum(cpqRackPowerSupplyCurPwrOutput{job="blade-oa/snmp"}) / 230) > 6.5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Blade: Power Usage Too High"
|
||||
description: "Power usage of blade system has been too high for last 10 minutes ({{ $value }}). https://grafana.int.lukegb.com/d/g-u3XQ8Gk/blade-power"
|
||||
- alert: BladePowerUsageOutOfBounds
|
||||
expr: node_hwmon_power_average_watt{system=~"blade-.*"} > on () group_left() (1.5 * quantile(0.5, node_hwmon_power_average_watt{system=~"blade-.*"}))
|
||||
for: 60m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Blade: Single Blade Power Usage Out of Bounds"
|
||||
description: "{{ $labels.system }} has power usage of {{ $value }}, which is out of expected bounds."
|
||||
|
||||
# Systems
|
||||
- alert: NodeExporterDown
|
||||
expr: up{exporter="node", system=~"(blade-(tuvok|paris|janeway|torres)|kusakabe|swann|totoro|clouvider-.*|etheroute-.*|bvm-.*)"} < 1
|
||||
expr: up{exporter="node", system=~"(rexxar|kusakabe|swann|totoro|clouvider-.*|etheroute-.*|bvm-.*)"} < 1
|
||||
for: 30m
|
||||
labels:
|
||||
severity: page
|
||||
|
|
|
|||
Loading…
Reference in a new issue