nixos-router/tests/basic.nix

# Single WAN + single LAN: DHCP, DNS, NAT, SLAAC and the default firewall
# policy for both address families. `wan4` selects how the IPv4 uplink works:
#
#   dhcp  - DHCPv4 on the uplink (ISP 203.0.113.1)
#   pppoe - PPPoE over VLAN 7 on the uplink (ISP 198.51.100.1, PPP MTU 1492)
#
# IPv6 is native and untagged on the uplink in both variants.
#
#   upstream (ISP) -- vlan 1 -- router -- vlan 2 -- client
#
# The fake ISP mimics the real uplink: RAs carry only a default route (no
# prefix to autoconfigure, no DHCPv6 flags), and Kea hands out one address
# plus a single delegated /64 out of 2001:db8:2::/48.
{
  self,
  pkgs,
  wan4 ? "dhcp",
}:
let
  pppoe = wan4 == "pppoe";
  ispV4 = if pppoe then "198.51.100.1" else "203.0.113.1";
  routerV4 = if pppoe then "198.51.100.100" else "203.0.113.100";
  pppoeVlan = 7;

  inherit (pkgs) lib;
  # Indent every line but the first (the interpolation site supplies that one).
  indent =
    n: text:
    lib.concatStringsSep "\n${lib.fixedWidthString n " " ""}" (
      lib.splitString "\n" (lib.removeSuffix "\n" text)
    );

  uplinkWait =
    if pppoe then
      ''
        upstream.wait_for_unit("pppoe-server.service")
        router.wait_for_unit("router-pppoe.service")
        router.wait_until_succeeds("ip -4 addr show dev pppoe-wan | grep -q ${routerV4}")
        router.wait_until_succeeds("ip -4 route show default | grep -q 'dev pppoe-wan'")
        # The ISP's route to the LAN needs the session to exist first.
        upstream.wait_until_succeeds("ip -4 addr show | grep -q ${ispV4}")
        upstream.succeed("ip route replace 192.168.10.0/24 via ${routerV4}")
      ''
    else
      ''
        router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q ${routerV4}")
      '';

  mssSubtest = ''
    with subtest("PPP link has MTU 1492 and forwarded TCP MSS is clamped"):
        assert "mtu 1492" in router.succeed("ip link show pppoe-wan"), "unexpected PPP MTU"
        # Capture the SYN as it leaves the router; the client itself offers MSS 1460.
        router.succeed("tcpdump -nn -i pppoe-wan -c 1 -w /tmp/syn.pcap 'tcp[tcpflags] & (tcp-syn|tcp-ack) = tcp-syn' >/dev/null 2>&1 &")
        router.wait_until_succeeds("pgrep tcpdump")
        router.sleep(1)
        client.succeed("curl -sf --max-time 5 --resolve example.test:80:${ispV4} http://example.test/")
        mss = router.wait_until_succeeds("tcpdump -nn -v -r /tmp/syn.pcap 2>&1 | grep -o 'mss [0-9]*'")
        assert mss.strip() == "mss 1452", f"expected the SYN clamped to mss 1452, got {mss!r}"
  '';

  # `fail` in the test script also passes on "command not found", so make sure
  # the tools the script relies on are really there.
  tools = { pkgs, ... }: {
    environment.systemPackages = with pkgs; [
      dnsutils
      netcat-openbsd
      curl
      tcpdump
    ];
  };
in
pkgs.testers.runNixOSTest {
  name = "router-${if pppoe then "pppoe" else "basic"}";

  defaults.imports = [ tools ];

  # `driverInteractive` uses the full QEMU, which would open a window per VM;
  # `nix run .#lab` wants the consoles in the terminal instead.
  interactive.defaults.virtualisation.qemu.options = [
    "-display"
    "none"
  ];

  nodes = {
    # Plays the ISP: hands out one WAN lease (v4 and v6) plus a delegated /64,
    # answers DNS, and serves a page that echoes the client address it sees
    # (to prove NAT happens for v4 and does not for v6).
    upstream =
      { lib, pkgs, ... }:
      lib.mkMerge [
        {
          virtualisation.vlans = [ 1 ];
          networking.useNetworkd = true;
          networking.useDHCP = false;
          networking.firewall.enable = false;
          networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];

          systemd.network.networks."10-wan" = {
            matchConfig.Name = "eth1";
            address = [ "2001:db8:1::1/64" ] ++ lib.optional (!pppoe) "203.0.113.1/24";
            # Routes back to the LAN via the router so that "unreachable" below is
            # due to the firewall and not a missing route. (With PPPoE the IPv4
            # route can only exist once the session is up; the test adds it.)
            routes = [
              {
                Destination = "2001:db8:2::/48";
                Gateway = "2001:db8:1::100";
              }
            ]
            ++ lib.optional (!pppoe) {
              Destination = "192.168.10.0/24";
              Gateway = "203.0.113.100";
            };
            networkConfig = {
              IPv6SendRA = true;
            }
            // lib.optionalAttrs pppoe { VLAN = [ "eth1.${toString pppoeVlan}" ]; };
            ipv6SendRAConfig = {
              Managed = false;
              OtherInformation = false;
              RouterLifetimeSec = 1800;
              EmitDNS = false;
            };
            # On-link only: the router must not autoconfigure a second address.
            ipv6Prefixes = [
              {
                Prefix = "2001:db8:1::/64";
                OnLink = true;
                AddressAutoconfiguration = false;
              }
            ];
          };

          services.dnsmasq = {
            enable = true;
            settings = {
              address = "/example.test/${ispV4}";
            }
            // (
              if pppoe then
                # The PPP interface only exists once a session is up.
                { bind-dynamic = true; }
              else
                {
                  interface = "eth1";
                  bind-interfaces = true;
                  dhcp-range = "203.0.113.100,203.0.113.100,255.255.255.0,1h";
                  dhcp-option = [ "option:router,203.0.113.1" ];
                }
            );
          };

          services.kea.dhcp6 = {
            enable = true;
            settings = {
              interfaces-config = {
                interfaces = [ "eth1" ];
                re-detect = true;
              };
              lease-database = {
                type = "memfile";
                persist = false;
              };
              preferred-lifetime = 3000;
              valid-lifetime = 4000;
              subnet6 = [
                {
                  id = 1;
                  subnet = "2001:db8:1::/64";
                  interface = "eth1";
                  pools = [ { pool = "2001:db8:1::100 - 2001:db8:1::100"; } ];
                  pd-pools = [
                    {
                      prefix = "2001:db8:2::";
                      prefix-len = 48;
                      delegated-len = 64;
                    }
                  ];
                }
              ];
            };
          };

          services.nginx = {
            enable = true;
            virtualHosts."example.test".locations."/".return = ''200 "$remote_addr"'';
          };
        }
        (lib.mkIf pppoe {
          boot.kernelModules = [
            "pppoe"
            "pppox"
            "ppp_generic"
          ];
          systemd.network.netdevs."05-pppoe" = {
            netdevConfig = {
              Kind = "vlan";
              Name = "eth1.${toString pppoeVlan}";
            };
            vlanConfig.Id = pppoeVlan;
          };
          systemd.network.networks."11-pppoe" = {
            matchConfig.Name = "eth1.${toString pppoeVlan}";
            networkConfig.LinkLocalAddressing = "no";
          };

          environment.etc = {
            "ppp/options".text = "";
            "ppp/chap-secrets".text = ''
              "testuser" * "p4ss\"w0rd" *
            '';
            "ppp/pppoe-server-options".text = ''
              require-chap
              noccp
              mtu 1492
              mru 1492
              lcp-echo-interval 10
              lcp-echo-failure 3
            '';
          };

          # The ISP's access concentrator: one session, 198.51.100.1 <-> .100.
          systemd.services.pppoe-server = {
            wantedBy = [ "multi-user.target" ];
            wants = [ "sys-subsystem-net-devices-eth1.${toString pppoeVlan}.device" ];
            after = [
              "systemd-networkd.service"
              "sys-subsystem-net-devices-eth1.${toString pppoeVlan}.device"
            ];
            serviceConfig = {
              ExecStart = lib.concatStringsSep " " [
                "${pkgs.rp-pppoe}/bin/pppoe-server -F -I eth1.${toString pppoeVlan}"
                "-L 198.51.100.1 -R 198.51.100.100 -N 1"
                "-O /etc/ppp/pppoe-server-options -q ${pkgs.ppp}/bin/pppd"
              ];
              Restart = "always";
            };
          };
        })
      ];

    router = { lib, ... }: {
      imports = [ self.nixosModules.router ];
      virtualisation.vlans = [
        1
        2
      ];

      environment.etc."pppoe-test-password" = {
        text = "p4ss\"w0rd";
        mode = "0600";
      };

      # The test driver preconfigures static addresses on every vlan interface.
      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
      networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ ];

      router = {
        enable = true;
        wan = {
          interface = "eth1";
          ipv6.enable = true;
          ipv4 = {
            type = wan4;
          }
          // lib.optionalAttrs pppoe {
            pppoe = {
              vlan = pppoeVlan;
              username = "testuser";
              passwordFile = "/etc/pppoe-test-password";
            };
          };
        };
        lans.main = {
          interface = "eth2";
          ipv4 = {
            address = "192.168.10.1";
          };
          ipv6.pd = true;
          dhcp.pool = {
            start = "192.168.10.100";
            end = "192.168.10.199";
          };
        };
        dns.forwarders = [ ispV4 ];
      };

      # Unbound answers `.test` locally (RFC 6761) and would validate DNSSEC
      # against a fake ISP resolver that cannot provide it; neither applies to
      # a real forwarder, so relax both for the made-up test domain only.
      services.unbound.settings.server = {
        local-zone = [ "test. transparent" ];
        domain-insecure = [ "example.test" ];
      };
    };

    client = { lib, ... }: {
      virtualisation.vlans = [ 2 ];
      networking.useNetworkd = true;
      networking.useDHCP = false;
      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
      systemd.network.networks."10-lan" = {
        matchConfig.Name = "eth1";
        networkConfig.DHCP = "ipv4";
      };
      networking.firewall.enable = false;
    };
  };

  testScript = ''
    start_all()

    upstream.wait_for_unit("dnsmasq.service")
    upstream.wait_for_unit("nginx.service")

    with subtest("router obtains its IPv4 uplink and brings up services"):
        router.wait_for_unit("network-online.target")
        ${indent 4 uplinkWait}
        router.wait_for_unit("nftables.service")
        router.wait_for_unit("kea-dhcp4-server.service")
        router.wait_for_unit("unbound.service")

    with subtest("client gets address, gateway and DNS from the router"):
        client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q 192.168.10.")
        client.succeed("ip route show default | grep -q 'via 192.168.10.1'")
        client.succeed("resolvectl status eth1 | grep -q 192.168.10.1")

    with subtest("client resolves names through the router"):
        client.wait_until_succeeds("dig +short +time=2 +tries=1 @192.168.10.1 example.test | grep -qx ${ispV4}")

    with subtest("client reaches the 'internet' and is NATed"):
        out = client.wait_until_succeeds("curl -sf --max-time 5 --resolve example.test:80:${ispV4} http://example.test/")
        assert out.strip() == "${routerV4}", f"expected NATed source ${routerV4}, got {out!r}"

    ${lib.optionalString pppoe (indent 0 mssSubtest)}
    with subtest("WAN cannot initiate to the router or the LAN"):
        # Positive control: the probe tool works against a port that is open.
        upstream.succeed("nc -z -w 2 ${ispV4} 80")
        upstream.fail("nc -z -w 2 ${routerV4} 22")
        upstream.fail("nc -z -w 2 ${routerV4} 53")
        upstream.fail("ping -c 1 -W 2 192.168.10.1")
        upstream.fail("ping -c 1 -W 2 ${routerV4}")
        client_ip = client.succeed("ip -4 -o addr show dev eth1 | awk '{print $4}' | cut -d/ -f1").strip()
        upstream.fail(f"ping -c 1 -W 2 {client_ip}")

    with subtest("router gets an IPv6 address, the delegated /64 and a default route"):
        router.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:1::100")
        router.wait_until_succeeds("ip -6 addr show dev eth2 scope global | grep -q 2001:db8:2:")
        router.wait_until_succeeds("ip -6 route show default | grep -q 'via fe80:.* dev eth1'")
        # The WAN prefix is not autoconfigured, so DHCPv6 is the only global address there.
        assert router.succeed("ip -6 -o addr show dev eth1 scope global | wc -l").strip() == "1"

    with subtest("client autoconfigures a global address from the delegated prefix"):
        client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:")
        client.wait_until_succeeds("ip -6 route show default | grep -q 'dev eth1'")
        client_v6 = client.succeed("ip -6 -o addr show dev eth1 scope global | awk '{print $4}' | cut -d/ -f1 | head -1").strip()

    with subtest("IPv6 is routed without NAT"):
        client.wait_until_succeeds("ping -6 -c 1 -W 2 2001:db8:1::1")
        out = client.succeed("curl -sf -g --max-time 5 -H 'Host: example.test' 'http://[2001:db8:1::1]/'")
        assert out.strip() == client_v6, f"expected the client's own address {client_v6}, got {out!r}"

    with subtest("WAN cannot initiate IPv6 to the router or the LAN"):
        # Positive control: the ISP really has a route to the client, so a
        # failure below is down to the firewall.
        upstream.succeed(f"ip -6 route get {client_v6} | grep -q 'via 2001:db8:1::100'")
        upstream.fail(f"ping -6 -c 1 -W 2 {client_v6}")
        upstream.fail("ping -6 -c 1 -W 2 2001:db8:1::100")
        upstream.fail("nc -6 -z -w 2 2001:db8:1::100 22")

    with subtest("LAN can reach SSH and ping on the router"):
        client.succeed("nc -z -w 2 192.168.10.1 22")
        client.succeed("ping -c 1 -W 2 192.168.10.1")
  '';
}