From 75d3386cd2159586283f237f07957f9ea3c23156 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Fri, 15 Apr 2022 23:33:53 +0100 Subject: [PATCH] treewide: fix up for nixpkgs bump --- ops/nixos/bvm-paperless/default.nix | 5 +- ops/nixos/lib/common.nix | 12 + .../patches/networkd-global-options.patch | 232 ------------- .../networkd-support-more-wg-options.patch | 21 -- third_party/nixpkgs/patches/pr167221.patch | 327 ------------------ third_party/nixpkgs/patches/series | 3 - 6 files changed, 14 insertions(+), 586 deletions(-) delete mode 100644 third_party/nixpkgs/patches/networkd-global-options.patch delete mode 100644 third_party/nixpkgs/patches/networkd-support-more-wg-options.patch delete mode 100644 third_party/nixpkgs/patches/pr167221.patch diff --git a/ops/nixos/bvm-paperless/default.nix b/ops/nixos/bvm-paperless/default.nix index 1be6af71d1..1f01106895 100644 --- a/ops/nixos/bvm-paperless/default.nix +++ b/ops/nixos/bvm-paperless/default.nix @@ -21,9 +21,8 @@ in { }; my.ip.tailscale = "100.85.236.121"; - services.paperless-ng = { + services.paperless = { enable = true; - package = pkgs.paperless-ngx; address = config.my.ip.tailscale; extraConfig = { PAPERLESS_ALLOWED_HOSTS = "paperless.int.lukegb.com,bvm-paperless.int.as205479.net:28981,bvm-paperless.int.as205479.net"; @@ -32,7 +31,7 @@ in { PAPERLESS_DBHOST = "localhost"; }; }; - systemd.services.paperless-ng-server.serviceConfig = { + systemd.services.paperless-scheduler.serviceConfig = { RestrictAddressFamilies = lib.mkForce []; SystemCallFilter = lib.mkForce []; PrivateNetwork = lib.mkForce false; diff --git a/ops/nixos/lib/common.nix b/ops/nixos/lib/common.nix index b12b86885d..f016ec4b74 100644 --- a/ops/nixos/lib/common.nix +++ b/ops/nixos/lib/common.nix @@ -199,6 +199,18 @@ in programs.mtr.enable = true; services.openssh.enable = true; + programs.ssh = { + extraConfig = '' + CanonicalizeHostname yes + CanonicalDomains int.as205479.net as205479.net + CanonicalizeMaxDots 0 + CanonicalizePermittedCNAMEs *.lukegb.com:*.as205479.net,*.int.as205479.net *.lukegb.dev:*.as205479.net,*.int.as205479.net *.zxcvbnm.ninja:*.as205479.net,*.int.as205479.net + ''; + knownHosts."*" = { + certAuthority = true; + publicKey = builtins.readFile ../../secrets/server-ca.pub; + }; + }; services.tailscale.enable = true; networking.firewall.interfaces.tailscale0 = { # Just allow anything in on tailscale0. diff --git a/third_party/nixpkgs/patches/networkd-global-options.patch b/third_party/nixpkgs/patches/networkd-global-options.patch deleted file mode 100644 index 65b37e6fa8..0000000000 --- a/third_party/nixpkgs/patches/networkd-global-options.patch +++ /dev/null @@ -1,232 +0,0 @@ -diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix ---- a/nixos/modules/system/boot/networkd.nix -+++ b/nixos/modules/system/boot/networkd.nix -@@ -10,6 +10,36 @@ let - - check = { - -+ global = { -+ sectionNetwork = checkUnitConfig "Network" [ -+ (assertOnlyFields [ -+ "SpeedMeter" -+ "SpeedMeterIntervalSec" -+ "ManageForeignRoutingPolicyRules" -+ "ManageForeignRoutes" -+ "RouteTable" -+ ]) -+ (assertValueOneOf "SpeedMeter" boolValues) -+ (assertInt "SpeedMeterIntervalSec") -+ (assertValueOneOf "ManageForeignRoutingPolicyRules" boolValues) -+ (assertValueOneOf "ManageForeignRoutes" boolValues) -+ ]; -+ -+ sectionDHCPv4 = checkUnitConfig "DHCPv4" [ -+ (assertOnlyFields [ -+ "DUIDType" -+ "DUIDRawData" -+ ]) -+ ]; -+ -+ sectionDHCPv6 = checkUnitConfig "DHCPv6" [ -+ (assertOnlyFields [ -+ "DUIDType" -+ "DUIDRawData" -+ ]) -+ ]; -+ }; -+ - link = { - - sectionLink = checkUnitConfig "Link" [ -@@ -871,6 +901,44 @@ let - }; - }; - -+ networkdOptions = { -+ networkConfig = mkOption { -+ default = {}; -+ example = { SpeedMeter = true; ManageForeignRoutingPolicyRules = false; }; -+ type = types.addCheck (types.attrsOf unitOption) check.global.sectionNetwork; -+ description = '' -+ Each attribute in this set specifies an option in the -+ [Network] section of the networkd config. -+ See networkd.conf -+ 5 for details. -+ ''; -+ }; -+ -+ dhcpV4Config = mkOption { -+ default = {}; -+ example = { DUIDType = "vendor"; }; -+ type = types.addCheck (types.attrsOf unitOption) check.global.sectionDHCPv4; -+ description = '' -+ Each attribute in this set specifies an option in the -+ [DHCPv4] section of the networkd config. -+ See networkd.conf -+ 5 for details. -+ ''; -+ }; -+ -+ dhcpV6Config = mkOption { -+ default = {}; -+ example = { DUIDType = "vendor"; }; -+ type = types.addCheck (types.attrsOf unitOption) check.global.sectionDHCPv6; -+ description = '' -+ Each attribute in this set specifies an option in the -+ [DHCPv6] section of the networkd config. -+ See networkd.conf -+ 5 for details. -+ ''; -+ }; -+ }; -+ - linkOptions = commonNetworkOptions // { - # overwrite enable option from above - enable = mkOption { -@@ -1519,6 +1587,39 @@ let - }; - }; - -+ networkdConfig = { config, ... }: { -+ options = { -+ routeTables = mkOption { -+ default = {}; -+ example = { foo = 27; }; -+ type = with types; attrsOf int; -+ description = '' -+ Defines route table names as an attrset of name to number. -+ See networkd.conf -+ 5 for details. -+ ''; -+ }; -+ -+ addRouteTablesToIPRoute2 = mkOption { -+ default = true; -+ example = false; -+ type = types.bool; -+ description = '' -+ If true and routeTables are set, then the specified route tables -+ will also be installed into /etc/iproute2/rt_tables. -+ ''; -+ }; -+ }; -+ -+ config = { -+ networkConfig = optionalAttrs (config.routeTables != { }) { -+ RouteTable = mapAttrsToList -+ (name: number: "${name}:${toString number}") -+ config.routeTables; -+ }; -+ }; -+ }; -+ - commonMatchText = def: optionalString (def.matchConfig != { }) '' - [Match] - ${attrsToSection def.matchConfig} -@@ -1600,6 +1701,20 @@ let - + def.extraConfig; - }; - -+ renderConfig = def: -+ { text = '' -+ [Network] -+ '' -+ + attrsToSection def.networkConfig -+ + optionalString (def.dhcpV4Config != { }) '' -+ [DHCPv4] -+ ${attrsToSection def.dhcpV4Config} -+ '' -+ + optionalString (def.dhcpV6Config != { }) '' -+ [DHCPv6] -+ ${attrsToSection def.dhcpV6Config} -+ ''; }; -+ - networkToUnit = name: def: - { inherit (def) enable; - text = commonMatchText def -@@ -1700,7 +1815,8 @@ let - unitFiles = listToAttrs (map (name: { - name = "systemd/network/${name}"; - value.source = "${cfg.units.${name}.unit}/${name}"; -- }) (attrNames cfg.units)); -+ }) (attrNames cfg.units)) // { -+ }; - in - - { -@@ -1732,6 +1848,12 @@ in - description = "Definition of systemd networks."; - }; - -+ systemd.network.config = mkOption { -+ default = {}; -+ type = with types; submodule [ { options = networkdOptions; } networkdConfig ]; -+ description = "Definition of global systemd network config."; -+ }; -+ - systemd.network.units = mkOption { - description = "Definition of networkd units."; - default = {}; -@@ -1776,7 +1898,9 @@ in - systemd.services.systemd-networkd = { - wantedBy = [ "multi-user.target" ]; - aliases = [ "dbus-org.freedesktop.network1.service" ]; -- restartTriggers = map (x: x.source) (attrValues unitFiles); -+ restartTriggers = map (x: x.source) (attrValues unitFiles) ++ [ -+ config.environment.etc."systemd/networkd.conf".source -+ ]; - }; - - systemd.services.systemd-networkd-wait-online = { -@@ -1795,6 +1919,17 @@ in - }; - }; - -+ environment.etc."systemd/networkd.conf" = renderConfig cfg.config; -+ -+ networking.iproute2 = mkIf (cfg.config.addRouteTablesToIPRoute2 && cfg.config.routeTables != { }) { -+ enable = mkDefault true; -+ rttablesExtraConfig = '' -+ -+ # Extra tables defined in NixOS systemd.networkd.config.routeTables. -+ ${concatStringsSep "\n" (mapAttrsToList (name: number: "${toString number} ${name}") cfg.config.routeTables)} -+ ''; -+ }; -+ - services.resolved.enable = mkDefault true; - }) - ]; -diff --git a/nixos/tests/systemd-networkd.nix b/nixos/tests/systemd-networkd.nix ---- a/nixos/tests/systemd-networkd.nix -+++ b/nixos/tests/systemd-networkd.nix -@@ -8,6 +8,9 @@ let generateNodeConf = { lib, pkgs, conf - environment.systemPackages = with pkgs; [ wireguard-tools ]; - systemd.network = { - enable = true; -+ config = { -+ routeTables.custom = 23; -+ }; - netdevs = { - "90-wg0" = { - netdevConfig = { Kind = "wireguard"; Name = "wg0"; }; -@@ -39,6 +42,7 @@ let generateNodeConf = { lib, pkgs, conf - address = [ "10.0.0.${nodeId}/32" ]; - routes = [ - { routeConfig = { Gateway = "10.0.0.${nodeId}"; Destination = "10.0.0.0/24"; }; } -+ { routeConfig = { Gateway = "10.0.0.${nodeId}"; Destination = "10.0.0.0/24"; Table = "custom"; }; } - ]; - }; - "30-eth1" = { -@@ -88,6 +92,12 @@ testScript = '' - node2.wait_for_unit("systemd-networkd-wait-online.service") - - # ================================ -+ # Networkd Config -+ # ================================ -+ node1.succeed("grep RouteTable=custom:23 /etc/systemd/networkd.conf") -+ node1.succeed("sudo ip route show table custom | grep '10.0.0.0/24 via 10.0.0.1 dev wg0 proto static'") -+ -+ # ================================ - # Wireguard - # ================================ - node1.succeed("ping -c 5 10.0.0.2") diff --git a/third_party/nixpkgs/patches/networkd-support-more-wg-options.patch b/third_party/nixpkgs/patches/networkd-support-more-wg-options.patch deleted file mode 100644 index 929699c8d3..0000000000 --- a/third_party/nixpkgs/patches/networkd-support-more-wg-options.patch +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix ---- a/nixos/modules/system/boot/networkd.nix -+++ b/nixos/modules/system/boot/networkd.nix -@@ -281,6 +281,8 @@ let - "PrivateKeyFile" - "ListenPort" - "FirewallMark" -+ "RouteTable" -+ "RouteMetric" - ]) - (assertInt "FirewallMark") - (assertRange "FirewallMark" 1 4294967295) -@@ -296,6 +298,8 @@ let - "AllowedIPs" - "Endpoint" - "PersistentKeepalive" -+ "RouteTable" -+ "RouteMetric" - ]) - (assertInt "PersistentKeepalive") - (assertRange "PersistentKeepalive" 0 65535) diff --git a/third_party/nixpkgs/patches/pr167221.patch b/third_party/nixpkgs/patches/pr167221.patch deleted file mode 100644 index 080310712e..0000000000 --- a/third_party/nixpkgs/patches/pr167221.patch +++ /dev/null @@ -1,327 +0,0 @@ -From 2c60fcf22061dc2e44502667224bc3d448dcbabc Mon Sep 17 00:00:00 2001 -From: Luke Granger-Brown -Date: Mon, 4 Apr 2022 18:30:41 +0000 -Subject: [PATCH 1/2] paperless-ngx: init at 1.6.0 - ---- - .../office/paperless-ngx/default.nix | 193 ++++++++++++++++++ - pkgs/top-level/all-packages.nix | 2 + - 2 files changed, 195 insertions(+) - create mode 100644 pkgs/applications/office/paperless-ngx/default.nix - -diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix -new file mode 100644 -index 0000000000000..2abf66f1e8362 ---- /dev/null -+++ b/pkgs/applications/office/paperless-ngx/default.nix -@@ -0,0 +1,193 @@ -+{ lib -+, fetchurl -+, fetchpatch -+, nixosTests -+, python3 -+, ghostscript -+, imagemagick -+, jbig2enc -+, optipng -+, pngquant -+, qpdf -+, tesseract4 -+, unpaper -+, liberation_ttf -+}: -+ -+let -+ py = python3.override { -+ packageOverrides = self: super: { -+ django = super.django_3; -+ -+ # Incompatible with aioredis 2 -+ aioredis = super.aioredis.overridePythonAttrs (oldAttrs: rec { -+ version = "1.3.1"; -+ src = oldAttrs.src.override { -+ inherit version; -+ sha256 = "0fi7jd5hlx8cnv1m97kv9hc4ih4l8v15wzkqwsp73is4n0qazy0m"; -+ }; -+ }); -+ }; -+ }; -+ -+ path = lib.makeBinPath [ ghostscript imagemagick jbig2enc optipng pngquant qpdf tesseract4 unpaper ]; -+in -+py.pkgs.pythonPackages.buildPythonApplication rec { -+ pname = "paperless-ngx"; -+ version = "1.6.0"; -+ -+ src = fetchurl { -+ url = "https://github.com/paperless-ngx/paperless-ngx/releases/download/ngx-${version}/${pname}-${version}.tar.xz"; -+ sha256 = "07mrxbwahkm00n9nvssd6d13p80w333g84cd38bzp0l34nzim5zl"; -+ }; -+ -+ format = "other"; -+ -+ # Make bind address configurable -+ postPatch = '' -+ substituteInPlace gunicorn.conf.py --replace "bind = " "# bind = " -+ ''; -+ -+ propagatedBuildInputs = with py.pkgs.pythonPackages; [ -+ aioredis -+ arrow -+ asgiref -+ async-timeout -+ attrs -+ autobahn -+ automat -+ blessed -+ certifi -+ cffi -+ channels-redis -+ channels -+ chardet -+ click -+ coloredlogs -+ concurrent-log-handler -+ constantly -+ cryptography -+ daphne -+ dateparser -+ django-cors-headers -+ django-extensions -+ django-filter -+ django-picklefield -+ django-q -+ django -+ djangorestframework -+ filelock -+ fuzzywuzzy -+ gunicorn -+ h11 -+ hiredis -+ httptools -+ humanfriendly -+ hyperlink -+ idna -+ imap-tools -+ img2pdf -+ incremental -+ inotify-simple -+ inotifyrecursive -+ joblib -+ langdetect -+ lxml -+ msgpack -+ numpy -+ ocrmypdf -+ pathvalidate -+ pdfminer -+ pikepdf -+ pillow -+ pluggy -+ portalocker -+ psycopg2 -+ pyasn1-modules -+ pyasn1 -+ pycparser -+ pyopenssl -+ python-dateutil -+ python-dotenv -+ python-gnupg -+ python-Levenshtein -+ python_magic -+ pytz -+ pyyaml -+ redis -+ regex -+ reportlab -+ requests -+ scikit-learn -+ scipy -+ service-identity -+ six -+ sortedcontainers -+ sqlparse -+ threadpoolctl -+ tika -+ tqdm -+ twisted.extras.tls -+ txaio -+ tzlocal -+ urllib3 -+ uvicorn -+ uvloop -+ watchdog -+ watchgod -+ wcwidth -+ websockets -+ whitenoise -+ whoosh -+ zope_interface -+ ]; -+ -+ installPhase = '' -+ mkdir -p $out/lib -+ cp -r . $out/lib/paperless-ngx -+ chmod +x $out/lib/paperless-ngx/src/manage.py -+ makeWrapper $out/lib/paperless-ngx/src/manage.py $out/bin/paperless-ngx \ -+ --prefix PYTHONPATH : "$PYTHONPATH" \ -+ --prefix PATH : "${path}" -+ ln -s $out/lib/paperless-ngx $out/lib/paperless-ng -+ ln -s $out/bin/paperless-ngx $out/bin/paperless-ng -+ ''; -+ -+ checkInputs = with py.pkgs.pythonPackages; [ -+ pytest-django -+ pytest-env -+ pytest-sugar -+ pytest-xdist -+ factory_boy -+ pytestCheckHook -+ ]; -+ -+ pytestFlagsArray = [ "src" ]; -+ -+ # The tests require: -+ # - PATH with runtime binaries -+ # - A temporary HOME directory for gnupg -+ # - XDG_DATA_DIRS with test-specific fonts -+ preCheck = '' -+ export PATH="${path}:$PATH" -+ export HOME=$(mktemp -d) -+ export XDG_DATA_DIRS="${liberation_ttf}/share:$XDG_DATA_DIRS" -+ -+ # Disable unneeded code coverage test -+ substituteInPlace src/setup.cfg \ -+ --replace "--cov --cov-report=html" "" -+ ''; -+ -+ passthru = { -+ # PYTHONPATH of all dependencies used by the package -+ pythonPath = python3.pkgs.makePythonPath propagatedBuildInputs; -+ inherit path; -+ }; -+ -+ meta = with lib; { -+ description = "A supercharged version of paperless: scan, index, and archive all of your physical documents"; -+ homepage = "https://paperless-ngx.readthedocs.io/en/latest/"; -+ license = licenses.gpl3Only; -+ maintainers = with maintainers; [ lukegb ]; -+ }; -+} -diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix -index 65c5b9e133f04..367a8c95fbd8c 100644 ---- a/pkgs/top-level/all-packages.nix -+++ b/pkgs/top-level/all-packages.nix -@@ -8857,6 +8857,8 @@ with pkgs; - - paperless-ng = callPackage ../applications/office/paperless-ng { }; - -+ paperless-ngx = callPackage ../applications/office/paperless-ngx { }; -+ - paperwork = callPackage ../applications/office/paperwork/paperwork-gtk.nix { }; - - papertrail = callPackage ../tools/text/papertrail { }; - -From c6fe5d87b7525cbb2581a6a48bc20ef05483afa2 Mon Sep 17 00:00:00 2001 -From: Luke Granger-Brown -Date: Mon, 4 Apr 2022 18:31:19 +0000 -Subject: [PATCH 2/2] nixos/tests: add test for paperless-ngx using - paperless-ng module - ---- - nixos/modules/services/misc/paperless-ng.nix | 1 - - nixos/tests/all-tests.nix | 1 + - nixos/tests/paperless-ngx.nix | 46 +++++++++++++++++++ - .../office/paperless-ngx/default.nix | 2 + - 4 files changed, 49 insertions(+), 1 deletion(-) - create mode 100644 nixos/tests/paperless-ngx.nix - -diff --git a/nixos/modules/services/misc/paperless-ng.nix b/nixos/modules/services/misc/paperless-ng.nix -index 881fa93c04eed..716049070d11e 100644 ---- a/nixos/modules/services/misc/paperless-ng.nix -+++ b/nixos/modules/services/misc/paperless-ng.nix -@@ -53,7 +53,6 @@ let - PrivateNetwork = true; - PrivateTmp = true; - PrivateUsers = true; -- ProcSubset = "pid"; - ProtectClock = true; - # Breaks if the home dir of the user is in /home - # Also does not add much value in combination with the TemporaryFileSystem. -diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix -index dcbdf34e9441c..efffc6464f593 100644 ---- a/nixos/tests/all-tests.nix -+++ b/nixos/tests/all-tests.nix -@@ -399,6 +399,7 @@ in - pantalaimon = handleTest ./matrix/pantalaimon.nix {}; - pantheon = handleTest ./pantheon.nix {}; - paperless-ng = handleTest ./paperless-ng.nix {}; -+ paperless-ngx = handleTest ./paperless-ngx.nix {}; - parsedmarc = handleTest ./parsedmarc {}; - pdns-recursor = handleTest ./pdns-recursor.nix {}; - peerflix = handleTest ./peerflix.nix {}; -diff --git a/nixos/tests/paperless-ngx.nix b/nixos/tests/paperless-ngx.nix -new file mode 100644 -index 0000000000000..6752b6391e42c ---- /dev/null -+++ b/nixos/tests/paperless-ngx.nix -@@ -0,0 +1,46 @@ -+import ./make-test-python.nix ({ lib, ... }: { -+ name = "paperless-ngx"; -+ meta.maintainers = with lib.maintainers; [ lukegb ]; -+ -+ nodes.machine = { pkgs, ... }: { -+ environment.systemPackages = with pkgs; [ imagemagick jq ]; -+ services.paperless-ng = { -+ enable = true; -+ package = pkgs.paperless-ngx; -+ passwordFile = builtins.toFile "password" "admin"; -+ }; -+ }; -+ -+ testScript = '' -+ machine.wait_for_unit("paperless-ng-consumer.service") -+ -+ with subtest("Create test doc"): -+ machine.succeed( -+ "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black " -+ "-annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png" -+ ) -+ -+ with subtest("Web interface gets ready"): -+ machine.wait_for_unit("paperless-ng-web.service") -+ # Wait until server accepts connections -+ machine.wait_until_succeeds("curl -fs localhost:28981") -+ -+ with subtest("Create web test doc"): -+ machine.succeed( -+ "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black " -+ "-annotate +5+20 'hello web 16-10-2005' /tmp/webdoc.png" -+ ) -+ machine.wait_until_succeeds("curl -u admin:admin -F document=@/tmp/webdoc.png -fs localhost:28981/api/documents/post_document/") -+ -+ with subtest("Documents are consumed"): -+ machine.wait_until_succeeds( -+ "(($(curl -u admin:admin -fs localhost:28981/api/documents/ | jq .count) == 2))" -+ ) -+ assert "2005-10-16" in machine.succeed( -+ "curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'" -+ ) -+ assert "2005-10-16" in machine.succeed( -+ "curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[1] | .created'" -+ ) -+ ''; -+}) -diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix -index 2abf66f1e8362..02d93e3cc4ead 100644 ---- a/pkgs/applications/office/paperless-ngx/default.nix -+++ b/pkgs/applications/office/paperless-ngx/default.nix -@@ -182,6 +182,8 @@ py.pkgs.pythonPackages.buildPythonApplication rec { - # PYTHONPATH of all dependencies used by the package - pythonPath = python3.pkgs.makePythonPath propagatedBuildInputs; - inherit path; -+ -+ tests = { inherit (nixosTests) paperless-ngx; }; - }; - - meta = with lib; { diff --git a/third_party/nixpkgs/patches/series b/third_party/nixpkgs/patches/series index b81ed8d9b7..d5be45e01c 100644 --- a/third_party/nixpkgs/patches/series +++ b/third_party/nixpkgs/patches/series @@ -1,4 +1 @@ nvidia-sideband-socket.patch -pr167221.patch -networkd-support-more-wg-options.patch -networkd-global-options.patch