From d6638eb6630c58b4a90a7e8df6cdd84b4cd8b2d3 Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <hg@lukegb.com>
Date: Sun, 7 Jan 2024 17:04:42 +0000
Subject: [PATCH] totoro: fix and add blog post describing recovery steps

---
 ops/nixos/totoro/default.nix                  |  1 +
 .../posts/2024-01-07-nixos-stuck-boot.md      | 49 +++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 web/lukegbcom/posts/2024-01-07-nixos-stuck-boot.md

diff --git a/ops/nixos/totoro/default.nix b/ops/nixos/totoro/default.nix
index d4eafb6e99..7fa8128247 100644
--- a/ops/nixos/totoro/default.nix
+++ b/ops/nixos/totoro/default.nix
@@ -51,6 +51,7 @@ in {
       fsType = "vfat";
     };
   };
+  boot.zfs.requestEncryptionCredentials = false;
 
   # Use the systemd-boot EFI boot loader.
   boot.loader.systemd-boot.enable = true;
diff --git a/web/lukegbcom/posts/2024-01-07-nixos-stuck-boot.md b/web/lukegbcom/posts/2024-01-07-nixos-stuck-boot.md
new file mode 100644
index 0000000000..fd1466db76
--- /dev/null
+++ b/web/lukegbcom/posts/2024-01-07-nixos-stuck-boot.md
@@ -0,0 +1,49 @@
+---
+title: "NixOS: Stuck Boot (bad systemd unit)"
+date: 2024-01-07
+layout: Post
+---
+
+One of my systems at home runs NixOS and receives some (encrypted) backups via
+`zfs send`/`zfs recv` shenanigans.  I don't want to actually decrypt these at
+boot, but I forgot to set `boot.zfs.requestEncryptionCredentials` appropriate,
+so I got dropped into a systemd recovery prompt.
+
+---
+
+To fix this enough that the system would boot, I manually made sure that I had
+imported and loaded the necessary keys for the ZFS pool in question (named
+"tank") manually:
+
+```
+zpool import tank
+zfs load-key tank/enc
+```
+
+Because zfs-import-tank is configured as `Type=oneshot` and
+`RemainAfterExit=true`, the unit only needs to be marked as successful once,
+then we can reload back to the "broken" config, but the fact that the unit ran
+will be remembered.
+
+As such, to make the system finish booting enough that I could reliably switch
+to a new system with a fixed config, I copied the systemd unit to /tmp, and
+sneakily edited it to replace the `ExecStart` with
+`/nix/var/nix/profiles/system/sw/bin/true`, then bind-mounted the unit over the
+one in /etc:
+
+```
+cat /etc/systemd/system/zfs-import-tank.service > /tmp/zfs-import-tank.service
+vim /tmp/zfs-import-tank.service
+mount --bind /tmp/zfs-import-tank.service /etc/systemd/system/zfs-import-tank.service
+
+# Start our hacked-up zfs-import-tank service
+systemctl daemon-reload
+systemctl start zfs-import-tank
+
+# Revert the system to its prior state for cleanliness
+umount /etc/systemd/system/zfs-import-tank.service
+systemctl daemon-reload
+
+# Finish booting
+systemctl default
+```