nixos-router/tests/image.nix

# The appliance image: boot the raw disk image under UEFI and check what makes
# it an appliance (read-only verity store, volatile root, persistent /var) while
# the router itself runs. Then update it A/B: version 2 goes into the empty
# slot, boots and is blessed; a broken version 3 replaces version 1, fails its
# health check on every try and systemd-boot falls back to version 2.
{ self, pkgs }:
let
  inherit (pkgs) lib;
  common = import ./common.nix;
  keyring = import "${pkgs.path}/nixos/tests/common/gpg-keyring.nix" { inherit pkgs; };

  # The router as an image; v1 boots it, v2 and v3 only provide their updates.
  router =
    { lib, ... }:
    {
      imports = [
        self.nixosModules.router
        self.nixosModules.image
        common.tools
      ];

      # The VM boots the image's own disk, not the usual test plumbing.
      virtualisation.fileSystems = lib.mkVMOverride { };
      virtualisation.directBoot.enable = false;
      virtualisation.mountHostNixStore = false;
      virtualisation.useEFIBoot = true;
      virtualisation.vlans = [
        1
        2
      ];
      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
      networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ ];

      # Keep the disk (and what the test writes to it) small.
      router.image = {
        storeSize = "1G";
        storeVeritySize = "64M";
        tries = 2;
        update = {
          url = "http://192.168.10.2/";
          publicKey = "${keyring}/pubkey.gpg";
        };
      };

      router = {
        enable = true;
        wan.interface = "eth1";
        lans.main = {
          interface = "eth2";
          internet = true;
          ipv4.address = "192.168.10.1";
          dhcp.pool = {
            start = "192.168.10.100";
            end = "192.168.10.199";
          };
        };
      };
    };

  # An update server's directory: the updates plus signed checksums.
  site =
    name: updates:
    pkgs.runCommand "updates-${name}" { nativeBuildInputs = [ pkgs.gnupg ]; } ''
      mkdir -p $out
      cd $out
      for u in ${toString updates}; do cp $u/*.zst $u/*.efi .; done
      sha256sum *.zst *.efi > SHA256SUMS
      export GNUPGHOME=$(mktemp -d)
      cp -R ${keyring}/* $GNUPGHOME
      gpg --batch --sign --detach-sign --output SHA256SUMS.gpg SHA256SUMS
    '';
in
pkgs.testers.runNixOSTest {
  name = "router-image";

  nodes = {
    machine = {
      imports = [ router ];
      system.image.version = "1";
    };
    v2 = {
      imports = [ router ];
      system.image.version = "2";
    };
    # Never gets healthy.
    v3 = {
      imports = [ router ];
      system.image.version = "3";
      systemd.services.broken = {
        wantedBy = [ "multi-user.target" ];
        serviceConfig.Type = "oneshot";
        script = "exit 1";
      };
      router.image.healthChecks = [ "broken.service" ];
    };

    server =
      { lib, ... }:
      {
        virtualisation.vlans = [ 2 ];
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
          {
            address = "192.168.10.2";
            prefixLength = 24;
          }
        ];
        networking.firewall.allowedTCPPorts = [ 80 ];
        services.nginx = {
          enable = true;
          virtualHosts.updates = {
            default = true;
            root = "/srv/updates";
          };
        };
      };
  };

  testScript =
    { nodes, ... }:
    let
      v2 = site "v2" [ nodes.v2.system.build.update ];
      v3 = site "v3" [
        nodes.v2.system.build.update
        nodes.v3.system.build.update
      ];
    in
    ''
      import datetime as dt
      import os
      import re
      import subprocess
      import tempfile

      # Before NIX_DISK_IMAGE is set: it applies to every VM started later.
      server.start()

      disk = tempfile.NamedTemporaryFile()
      subprocess.run([
        "${nodes.machine.virtualisation.qemu.package}/bin/qemu-img", "create",
        "-f", "qcow2",
        "-b", "${nodes.machine.system.build.image}/${nodes.machine.image.filePath}",
        "-F", "raw", disk.name,
      ], check=True)
      os.environ["NIX_DISK_IMAGE"] = disk.name

      def labels():
          return machine.succeed("lsblk -n -o PARTLABEL /dev/vda").split()

      def booted_version():
          return machine.succeed(". /etc/os-release; echo $IMAGE_VERSION").strip()

      def usr_slot():
          """The label of the partition the verity /usr is read from."""
          devs = re.findall(r"\((\w+)\)", machine.succeed("dmsetup deps -o devname usr"))
          parts = [machine.succeed(f"lsblk -n -o PARTLABEL /dev/{d}").strip() for d in devs]
          [data] = [p for p in parts if p.startswith("store_")]
          return data

      def reboot():
          machine.shutdown()
          machine.start()
          machine.wait_for_unit("multi-user.target")

      machine.wait_for_unit("multi-user.target")

      with subtest("the disk has the appliance layout, slot B is empty"):
          parts = labels()
          for label in ("esp", "store-verity_1", "store_1", "state"):
              assert label in parts, parts
          assert parts.count("_empty") == 2, parts
          assert usr_slot() == "store_1"

      with subtest("root is volatile and /nix/store is a dm-verity protected read-only partition"):
          machine.succeed("findmnt --kernel --type tmpfs /")
          assert "ACTIVE" in machine.succeed("dmsetup info --target verity usr")
          backing = machine.succeed("df --output=source /nix/store | tail -n1").strip()
          assert backing == "/dev/mapper/usr", backing
          machine.fail("touch /nix/store/should-not-work")
          machine.fail("touch /etc/should-not-work")

      with subtest("the image identifies itself"):
          assert booted_version() == "1"
          machine.succeed("grep -qE '^IMAGE_ID=.?nixos-router.?$' /etc/os-release")

      with subtest("the router comes up from the image"):
          machine.wait_for_unit("nftables.service")
          machine.wait_for_unit("unbound.service")
          machine.wait_for_unit("kea-dhcp4-server.service")
          machine.wait_until_succeeds("ip -4 addr show dev eth2 | grep -q 192.168.10.1")

      with subtest("there is no Nix on the device"):
          machine.fail("command -v nix-env")

      with subtest("/var is on the state partition and survives a reboot, the root does not"):
          src = machine.succeed("findmnt -n -o SOURCE /var").strip()
          assert "vda" in src, src
          machine.succeed("echo kept > /var/lib/marker; echo lost > /root/marker; sync")
          reboot()
          assert machine.succeed("cat /var/lib/marker").strip() == "kept"
          machine.fail("test -e /root/marker")
          machine.wait_for_unit("nftables.service")

      server.wait_for_unit("nginx.service")

      with subtest("an update whose checksums do not match the signature is refused"):
          server.succeed("mkdir -p /srv/updates && cp ${v2}/* /srv/updates/")
          server.succeed("echo tampered >> /srv/updates/SHA256SUMS")
          machine.wait_until_succeeds("curl -sf http://192.168.10.2/SHA256SUMS")
          out = machine.fail("router-update update 2>&1")
          assert "Signature verification failed" in out, out
          assert labels().count("_empty") == 2

      with subtest("version 2 is installed into the empty slot"):
          server.succeed("rm -r /srv/updates && ln -s ${v2} /srv/updates")
          machine.succeed("router-update update >&2")
          parts = labels()
          assert "store_2" in parts and "store-verity_2" in parts, parts
          assert "store_1" in parts, parts
          machine.succeed("test -e /boot/EFI/Linux/nixos-router_2+2-0.efi")

      with subtest("version 2 boots from slot B, is healthy and gets blessed"):
          reboot()
          assert booted_version() == "2"
          assert usr_slot() == "store_2"
          machine.wait_for_unit("router-healthy.service")
          machine.wait_for_unit("systemd-bless-boot.service")
          machine.succeed("test -e /boot/EFI/Linux/nixos-router_2.efi")

      with subtest("version 3 replaces version 1, not the running version"):
          server.succeed("ln -sfn ${v3} /srv/updates")
          machine.succeed("router-update update >&2")
          parts = labels()
          assert "store_3" in parts and "store_2" in parts and "store_1" not in parts, parts
          machine.succeed("test -e /boot/EFI/Linux/nixos-router_3+2-0.efi")
          machine.fail("test -e /boot/EFI/Linux/nixos-router_1.efi")

      with subtest("version 3 never gets healthy and version 2 boots again"):
          machine.shutdown()
          # Each try of version 3 fails its health check and reboots, which
          # ends the VM (-no-reboot); start it again like a reset would.
          for _ in range(2):
              machine.start()
              machine.wait_for_shutdown(timeout=dt.timedelta(minutes=3))
          machine.start()
          machine.wait_for_unit("multi-user.target")
          assert booted_version() == "2"
          assert usr_slot() == "store_2"
          machine.succeed("test -e /boot/EFI/Linux/nixos-router_3+0-2.efi")
          machine.wait_for_unit("nftables.service")
    '';
}