nixos-router/tests/onestick.nix

# Router on a stick: one NIC carries the untagged LAN and, as VLAN 7, the
# PPPoE uplink, like a single-port box behind a VLAN-tagged ISP. IPv6 runs
# inside the PPP session: the ISP sends RAs and delegates a /64 on the PPP link.
#
#   upstream (ISP, PPPoE on vlan 7) -- switch -- router (eth1) -- client, peer
#
# The ISP and the LAN share one L2 segment in the test, which is what a real
# trunk looks like to the router: tagged frames go to the ISP, untagged to the LAN.
{ self, pkgs }:
let
  inherit (pkgs) lib;
  common = import ./common.nix;
  vlan = 7;
  ispV4 = "198.51.100.1";
  routerV4 = "198.51.100.100";

  lanHost =
    address:
    { lib, ... }:
    {
      virtualisation.vlans = [ 1 ];
      networking.useNetworkd = true;
      networking.useDHCP = false;
      networking.firewall.enable = false;
      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
      systemd.network.networks."10-lan" = {
        matchConfig.Name = "eth1";
        networkConfig.DHCP = if address == null then "ipv4" else "no";
        address = lib.optional (address != null) "${address}/24";
        gateway = lib.optional (address != null) "192.168.10.1";
      };
    };
in
pkgs.testers.runNixOSTest {
  name = "router-onestick";

  defaults.imports = [ common.tools ];
  interactive.defaults = common.interactiveDefaults;

  nodes = {
    upstream =
      { lib, pkgs, ... }:
      {
        virtualisation.vlans = [ 1 ];
        networking.useNetworkd = true;
        networking.useDHCP = false;
        networking.firewall.enable = false;
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
        boot.kernelModules = [
          "pppoe"
          "pppox"
          "ppp_generic"
        ];
        systemd.network.netdevs."05-pppoe" = {
          netdevConfig = {
            Kind = "vlan";
            Name = "eth1.${toString vlan}";
          };
          vlanConfig.Id = vlan;
        };
        systemd.network.networks."10-trunk" = {
          matchConfig.Name = "eth1";
          networkConfig = {
            VLAN = [ "eth1.${toString vlan}" ];
            LinkLocalAddressing = "no";
          };
        };
        systemd.network.networks."11-pppoe" = {
          matchConfig.Name = "eth1.${toString vlan}";
          networkConfig.LinkLocalAddressing = "no";
        };
        # The server's global address, reachable through the session.
        systemd.network.networks."05-lo" = {
          matchConfig.Name = "lo";
          address = [ "2001:db8:1::1/128" ];
        };
        # RAs and DHCPv6 prefix delegation on the (dynamic) PPP device. Kea
        # cannot serve a point-to-point link (it only sees the peer's address),
        # so a minimal responder stands in for the ISP's DHCPv6 server. The test
        # starts both once the session exists.
        environment.systemPackages = [
          pkgs.radvd
          pkgs.python3
        ];
        boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1;
        environment.etc."dhcp6-pd.py".text = ''
          import socket, struct, subprocess, sys

          PREFIX = socket.inet_pton(socket.AF_INET6, "2001:db8:2::")
          DEV = sys.argv[1]
          ifindex = socket.if_nametoindex(DEV)
          sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
          sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
          sock.setsockopt(socket.SOL_SOCKET, 25, DEV.encode() + b"\0")  # SO_BINDTODEVICE
          sock.bind(("::", 547))
          group = socket.inet_pton(socket.AF_INET6, "ff02::1:2")
          sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group + struct.pack("@I", ifindex))
          SERVER_ID = struct.pack("!HH", 3, 0) + b"\x00\x03\x00\x01" + bytes(6)

          def options(data):
              out, i = {}, 0
              while i + 4 <= len(data):
                  code, length = struct.unpack("!HH", data[i:i + 4])
                  out[code] = data[i + 4:i + 4 + length]
                  i += 4 + length
              return out

          def opt(code, body):
              return struct.pack("!HH", code, len(body)) + body

          print("dhcp6-pd: listening on", DEV, flush=True)
          while True:
              data, addr = sock.recvfrom(2048)
              mtype, xid, opts = data[0], data[1:4], options(data[4:])
              if mtype not in (1, 3) or 25 not in opts:  # Solicit, Request with IA_PD
                  continue
              iaid = opts[25][:4]
              prefix = opt(26, struct.pack("!IIB", 3000, 4000, 64) + PREFIX)
              ia_pd = opt(25, iaid + struct.pack("!II", 1800, 2700) + prefix)
              rapid = mtype == 1 and 14 in opts
              reply = bytes([7 if (mtype == 3 or rapid) else 2]) + xid
              reply += opt(1, opts[1]) + opt(2, SERVER_ID[4:]) + ia_pd
              if rapid:
                  reply += opt(14, b"")
              sock.sendto(reply, addr)
              print("dhcp6-pd: answered", mtype, "from", addr, flush=True)
        '';
        environment.etc = {
          "ppp/options".text = "";
          "ppp/chap-secrets".text = ''
            "testuser" * "p4ssw0rd" *
          '';
          "ppp/pppoe-server-options".text = ''
            require-chap
            +ipv6
            noccp
            mtu 1492
            mru 1492
          '';
        };
        systemd.services.pppoe-server = {
          wantedBy = [ "multi-user.target" ];
          wants = [ "sys-subsystem-net-devices-eth1.${toString vlan}.device" ];
          after = [
            "systemd-networkd.service"
            "sys-subsystem-net-devices-eth1.${toString vlan}.device"
          ];
          serviceConfig = {
            ExecStart = lib.concatStringsSep " " [
              "${pkgs.rp-pppoe}/bin/pppoe-server -F -I eth1.${toString vlan}"
              "-L ${ispV4} -R ${routerV4} -N 1"
              "-O /etc/ppp/pppoe-server-options -q ${pkgs.ppp}/bin/pppd"
            ];
            Restart = "always";
          };
        };
        services.nginx = {
          enable = true;
          virtualHosts."isp".locations."/".return = ''200 "$remote_addr"'';
        };
      };

    router =
      { lib, ... }:
      {
        imports = [ self.nixosModules.router ];
        virtualisation.vlans = [ 1 ];
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
        environment.etc."pppoe-test-password" = {
          text = "p4ssw0rd";
          mode = "0600";
        };
        router = {
          enable = true;
          wan = {
            interface = "eth1";
            ipv6 = {
              enable = true;
              transport = "ppp";
            };
            ipv4 = {
              type = "pppoe";
              pppoe = {
                inherit vlan;
                username = "testuser";
                passwordFile = "/etc/pppoe-test-password";
              };
            };
          };
          lans.main = {
            interface = "eth1";
            internet = true;
            ipv4.address = "192.168.10.1";
            ipv6.pd = true;
            ipv6.address = "fd2c:cdef:cde9::1/64";
            dhcp.pool = {
              start = "192.168.10.100";
              end = "192.168.10.199";
            };
          };
          # Only the PPP device is an uplink, so this must work from the ISP
          # and must not fire for traffic arriving from the LAN.
          firewall.rules = [
            {
              from = "wan";
              to = "main";
              destination = "192.168.10.51";
              protocol = "tcp";
              ports = [ 8080 ];
              dnat = true;
            }
          ];
          dns = {
            forwarders = [ ispV4 ];
            domain = "lan.test";
            hosts.nas = [ "192.168.10.20" ];
          };
        };
      };

    client = lanHost null;

    peer =
      { lib, ... }:
      {
        imports = [ (lanHost "192.168.10.51") ];
        services.nginx = {
          enable = true;
          virtualHosts."peer" = {
            listen = [
              {
                addr = "0.0.0.0";
                port = 8080;
              }
            ];
            locations."/".return = ''200 "$remote_addr"'';
          };
        };
      };
  };

  testScript = ''
    start_all()
    upstream.wait_for_unit("nginx.service")
    peer.wait_for_unit("nginx.service")

    with subtest("the PPPoE session comes up on the tagged VLAN of the shared link"):
        upstream.wait_for_unit("pppoe-server.service")
        router.wait_for_unit("router-pppoe.service")
        router.wait_until_succeeds("ip -4 addr show dev pppoe-wan | grep -q ${routerV4}")
        router.wait_until_succeeds("ip -4 route show default | grep -q 'dev pppoe-wan'")
        assert "vlan protocol 802.1Q id ${toString vlan}" in router.succeed("ip -d link show eth1.${toString vlan}")
        router.wait_for_unit("nftables.service")
        router.wait_for_unit("kea-dhcp4-server.service")

    with subtest("the untagged side of the same NIC is the LAN"):
        router.succeed("ip -4 addr show dev eth1 | grep -q 192.168.10.1/24")
        client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q 192.168.10.100")
        client.succeed("ip route show default | grep -q 'via 192.168.10.1'")

    with subtest("the LAN reaches the ISP through PPPoE and is NATed"):
        out = client.wait_until_succeeds("curl -sf --max-time 5 http://${ispV4}/")
        assert out.strip() == "${routerV4}", out

    with subtest("the ISP cannot reach the router or the LAN"):
        upstream.succeed("nc -z -w 2 ${ispV4} 80")
        upstream.fail("nc -z -w 2 ${routerV4} 22")
        upstream.fail("nc -z -w 2 ${routerV4} 53")
        upstream.fail("ping -c 1 -W 2 ${routerV4}")

    with subtest("a port forward works from the ISP"):
        upstream.succeed("ip route replace 192.168.10.0/24 via ${routerV4}")
        out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8080/")
        assert out.strip() == "${ispV4}", out

    with subtest("the shared NIC is not an uplink: LAN traffic never hits WAN rules"):
        # Positive control: the target answers directly.
        client.succeed("curl -sf --max-time 5 http://192.168.10.51:8080/")
        # Same port on the router's LAN address: no forward applies (and the
        # input chain would drop it anyway), so this must fail.
        client.fail("curl -sf --max-time 3 http://192.168.10.1:8080/")
        router.succeed("nft list ruleset | grep -q 'iifname \"pppoe-wan\"'")
        assert 'iifname { "eth1"' not in router.succeed("nft list ruleset"), "eth1 treated as an uplink"

    with subtest("IPv6 comes up inside the PPP session"):
        upstream.wait_until_succeeds("ip link show ppp0")
        upstream.wait_until_succeeds("ip -6 -o addr show dev ppp0 scope link | grep -q inet6")
        upstream.succeed("systemd-run --unit=dhcp6pd python3 /etc/dhcp6-pd.py ppp0")
        upstream.succeed("printf 'interface ppp0 {\\n AdvSendAdvert on;\\n AdvDefaultLifetime 1800;\\n};\\n' > /tmp/radvd.conf")
        upstream.succeed("systemd-run --unit=radvd6 radvd -n -C /tmp/radvd.conf")
        router.wait_until_succeeds("ip -6 route show default | grep -q 'dev pppoe-wan'")
        router.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:")
        # Nothing IPv6 on the physical link or the PPPoE VLAN.
        router.fail("ip -6 addr show dev eth1.${toString vlan} | grep -q inet6")
        ll = router.succeed("ip -6 -o addr show dev pppoe-wan scope link | awk '{print $4}' | cut -d/ -f1 | head -1").strip()
        assert ll.startswith("fe80:"), ll
        # The ISP routes the delegated prefix to the router's link-local address.
        upstream.succeed(f"ip -6 route replace 2001:db8:2::/48 via {ll} dev ppp0")

    with subtest("the client autoconfigures from the delegated prefix, without NAT"):
        client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:")
        client_v6 = client.succeed("ip -6 -o addr show dev eth1 scope global | awk '{print $4}' | cut -d/ -f1 | grep 2001:db8:2: | head -1").strip()
        client.wait_until_succeeds("ping -6 -c 1 -W 2 2001:db8:1::1")
        out = client.succeed("curl -sf -g --max-time 5 'http://[2001:db8:1::1]/'")
        assert out.strip() == client_v6, f"expected {client_v6}, got {out!r}"

    with subtest("IPv6 MSS is clamped to the PPP MTU"):
        router.succeed("tcpdump -nn -i pppoe-wan -c 1 -w /tmp/syn6.pcap 'ip6 and ip6[6] == 6 and ip6[53] & 0x12 == 0x02' >/dev/null 2>&1 &")
        router.wait_until_succeeds("pgrep tcpdump")
        router.sleep(1)
        client.succeed("curl -sf -g --max-time 5 'http://[2001:db8:1::1]/'")
        mss = router.wait_until_succeeds("tcpdump -nn -v -r /tmp/syn6.pcap 2>&1 | grep -o 'mss [0-9]*'")
        assert mss.strip() == "mss 1432", f"expected mss 1432, got {mss!r}"

    with subtest("the ISP cannot initiate IPv6 to the router or the LAN"):
        # Positive control: the ISP really has a route to the client.
        upstream.succeed(f"ip -6 route get {client_v6} | grep -q 'dev ppp0'")
        upstream.fail(f"ping -6 -c 1 -W 2 {client_v6}")
        upstream.fail(f"nc -6 -z -w 2 {client_v6} 22")
        upstream.fail(f"nc -6 -z -w 2 {ll}%ppp0 22")

    with subtest("the ULA is announced and the resolver answers on it"):
        router.succeed("ip -6 addr show dev eth1 | grep -q fd2c:cdef:cde9::1/64")
        client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q fd2c:cdef:cde9:")
        client.succeed("ping -6 -c 1 -W 2 fd2c:cdef:cde9::1")
        out = client.succeed("dig +short +time=2 +tries=1 @fd2c:cdef:cde9::1 nas.lan.test").strip()
        assert out == "192.168.10.20", out
        # The RA carries the ULA prefix (SLAAC, on-link) and the resolver as RDNSS.
        # A link bounce makes the client solicit one right away.
        router.succeed("(timeout 20 tcpdump -nn -vv -i eth1 -c 1 'icmp6 and ip6[40] == 134' > /tmp/ra.txt 2>&1 &)")
        router.wait_until_succeeds("pgrep tcpdump")
        router.sleep(1)
        client.succeed("ip link set eth1 down; ip link set eth1 up")
        ra = router.wait_until_succeeds("grep -q 'router lifetime' /tmp/ra.txt && cat /tmp/ra.txt")
        assert "fd2c:cdef:cde9::/64, Flags [onlink, auto]" in ra, ra
        assert "rdnss option" in ra and "fd2c:cdef:cde9::1" in ra, ra
  '';
}