# Single WAN + single LAN: DHCP, DNS, NAT, SLAAC and the default firewall # policy for both address families. `wan4` selects how the IPv4 uplink works: # # dhcp - DHCPv4 on the uplink (ISP 203.0.113.1) # pppoe - PPPoE over VLAN 7 on the uplink (ISP 198.51.100.1, PPP MTU 1492) # # IPv6 is native and untagged on the uplink in both variants. # # upstream (ISP) -- vlan 1 -- router -- vlan 2 -- client # # The fake ISP mimics the real uplink: RAs carry only a default route (no # prefix to autoconfigure, no DHCPv6 flags), and Kea hands out one address # plus a single delegated /64 out of 2001:db8:2::/48. { self, pkgs, wan4 ? "dhcp", }: let pppoe = wan4 == "pppoe"; ispV4 = if pppoe then "198.51.100.1" else "203.0.113.1"; routerV4 = if pppoe then "198.51.100.100" else "203.0.113.100"; pppoeVlan = 7; clientV4 = "192.168.10.100"; clientV6 = "2001:db8:2::1234"; peerV4 = "192.168.10.51"; inherit (pkgs) lib; # Indent every line but the first (the interpolation site supplies that one). indent = n: text: lib.concatStringsSep "\n${lib.fixedWidthString n " " ""}" ( lib.splitString "\n" (lib.removeSuffix "\n" text) ); uplinkWait = if pppoe then '' upstream.wait_for_unit("pppoe-server.service") router.wait_for_unit("router-pppoe.service") router.wait_until_succeeds("ip -4 addr show dev pppoe-wan | grep -q ${routerV4}") router.wait_until_succeeds("ip -4 route show default | grep -q 'dev pppoe-wan'") # The ISP's route to the LAN needs the session to exist first. upstream.wait_until_succeeds("ip -4 addr show | grep -q ${ispV4}") upstream.succeed("ip route replace 192.168.10.0/24 via ${routerV4}") '' else '' router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q ${routerV4}") ''; mssSubtest = '' with subtest("PPP link has MTU 1492 and forwarded TCP MSS is clamped"): assert "mtu 1492" in router.succeed("ip link show pppoe-wan"), "unexpected PPP MTU" # Capture the SYN as it leaves the router; the client itself offers MSS 1460. router.succeed("tcpdump -nn -i pppoe-wan -c 1 -w /tmp/syn.pcap 'tcp[tcpflags] & (tcp-syn|tcp-ack) = tcp-syn' >/dev/null 2>&1 &") router.wait_until_succeeds("pgrep tcpdump") router.sleep(1) client.succeed("curl -sf --max-time 5 --resolve example.test:80:${ispV4} http://example.test/") mss = router.wait_until_succeeds("tcpdump -nn -v -r /tmp/syn.pcap 2>&1 | grep -o 'mss [0-9]*'") assert mss.strip() == "mss 1452", f"expected the SYN clamped to mss 1452, got {mss!r}" ''; common = import ./common.nix; in pkgs.testers.runNixOSTest { name = "router-${if pppoe then "pppoe" else "basic"}"; defaults.imports = [ common.tools ]; interactive.defaults = common.interactiveDefaults; nodes = { # Plays the ISP: hands out one WAN lease (v4 and v6) plus a delegated /64, # answers DNS, and serves a page that echoes the client address it sees # (to prove NAT happens for v4 and does not for v6). upstream = { lib, pkgs, ... }: lib.mkMerge [ { virtualisation.vlans = [ 1 ]; networking.useNetworkd = true; networking.useDHCP = false; networking.firewall.enable = false; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; systemd.network.networks."10-wan" = { matchConfig.Name = "eth1"; address = [ "2001:db8:1::1/64" ] ++ lib.optional (!pppoe) "203.0.113.1/24"; # Routes back to the LAN via the router so that "unreachable" below is # due to the firewall and not a missing route. (With PPPoE the IPv4 # route can only exist once the session is up; the test adds it.) routes = [ { Destination = "2001:db8:2::/48"; Gateway = "2001:db8:1::100"; } ] ++ lib.optional (!pppoe) { Destination = "192.168.10.0/24"; Gateway = "203.0.113.100"; }; networkConfig = { IPv6SendRA = true; } // lib.optionalAttrs pppoe { VLAN = [ "eth1.${toString pppoeVlan}" ]; }; ipv6SendRAConfig = { Managed = false; OtherInformation = false; RouterLifetimeSec = 1800; EmitDNS = false; }; # On-link only: the router must not autoconfigure a second address. ipv6Prefixes = [ { Prefix = "2001:db8:1::/64"; OnLink = true; AddressAutoconfiguration = false; } ]; }; services.dnsmasq = { enable = true; settings = { address = "/example.test/${ispV4}"; } // ( if pppoe then # The PPP interface only exists once a session is up. { bind-dynamic = true; } else { interface = "eth1"; bind-interfaces = true; dhcp-range = "203.0.113.100,203.0.113.100,255.255.255.0,1h"; dhcp-option = [ "option:router,203.0.113.1" ]; } ); }; services.kea.dhcp6 = { enable = true; settings = { interfaces-config = { interfaces = [ "eth1" ]; re-detect = true; }; lease-database = { type = "memfile"; persist = false; }; preferred-lifetime = 3000; valid-lifetime = 4000; subnet6 = [ { id = 1; subnet = "2001:db8:1::/64"; interface = "eth1"; pools = [ { pool = "2001:db8:1::100 - 2001:db8:1::100"; } ]; pd-pools = [ { prefix = "2001:db8:2::"; prefix-len = 48; delegated-len = 64; } ]; } ]; }; }; services.nginx = { enable = true; virtualHosts."example.test".locations."/".return = ''200 "$remote_addr"''; }; } (lib.mkIf pppoe { boot.kernelModules = [ "pppoe" "pppox" "ppp_generic" ]; systemd.network.netdevs."05-pppoe" = { netdevConfig = { Kind = "vlan"; Name = "eth1.${toString pppoeVlan}"; }; vlanConfig.Id = pppoeVlan; }; systemd.network.networks."11-pppoe" = { matchConfig.Name = "eth1.${toString pppoeVlan}"; networkConfig.LinkLocalAddressing = "no"; }; environment.etc = { "ppp/options".text = ""; "ppp/chap-secrets".text = '' "testuser" * "p4ss\"w0rd" * ''; "ppp/pppoe-server-options".text = '' require-chap noccp mtu 1492 mru 1492 lcp-echo-interval 10 lcp-echo-failure 3 ''; }; # The ISP's access concentrator: one session, 198.51.100.1 <-> .100. systemd.services.pppoe-server = { wantedBy = [ "multi-user.target" ]; wants = [ "sys-subsystem-net-devices-eth1.${toString pppoeVlan}.device" ]; after = [ "systemd-networkd.service" "sys-subsystem-net-devices-eth1.${toString pppoeVlan}.device" ]; serviceConfig = { ExecStart = lib.concatStringsSep " " [ "${pkgs.rp-pppoe}/bin/pppoe-server -F -I eth1.${toString pppoeVlan}" "-L 198.51.100.1 -R 198.51.100.100 -N 1" "-O /etc/ppp/pppoe-server-options -q ${pkgs.ppp}/bin/pppd" ]; Restart = "always"; }; }; }) ]; router = { lib, ... }: { imports = [ self.nixosModules.router ]; virtualisation.vlans = [ 1 2 ]; environment.etc."pppoe-test-password" = { text = "p4ss\"w0rd"; mode = "0600"; }; # The test driver preconfigures static addresses on every vlan interface. networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ ]; router = { enable = true; wan = { interface = "eth1"; ipv6.enable = true; ipv4 = { type = wan4; } // lib.optionalAttrs pppoe { pppoe = { vlan = pppoeVlan; username = "testuser"; passwordFile = "/etc/pppoe-test-password"; }; }; }; lans.main = { interface = "eth2"; internet = true; ipv4 = { address = "192.168.10.1"; }; ipv6.pd = true; dhcp.pool = { start = "192.168.10.100"; end = "192.168.10.199"; }; }; dns.forwarders = [ ispV4 ]; firewall.rules = let fwd = r: { from = "wan"; to = "main"; destination = clientV4; protocol = "tcp"; dnat = true; } // r; in [ (fwd { ports = [ 8080 ]; hairpin = true; log = true; }) (fwd { ports = [ 8083 ]; toPort = 8084; }) (fwd { ports = [ "9000-9002" ]; }) (fwd { ports = [ 8085 ]; sources = [ "${ispV4}/32" ]; }) (fwd { ports = [ 8086 ]; sources = [ "192.0.2.0/24" ]; }) (fwd { ports = [ 8087 ]; rateLimit = "3/minute"; log = true; }) { from = "wan"; to = "main"; destination = clientV6; protocol = "tcp"; ports = [ 8081 ]; } ]; }; # Unbound answers `.test` locally (RFC 6761) and would validate DNSSEC # against a fake ISP resolver that cannot provide it; neither applies to # a real forwarder, so relax both for the made-up test domain only. services.unbound.settings.server = { local-zone = [ "test. transparent" ]; domain-insecure = [ "example.test" ]; }; }; client = { lib, ... }: { virtualisation.vlans = [ 2 ]; networking.useNetworkd = true; networking.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; systemd.network.networks."10-lan" = { matchConfig.Name = "eth1"; networkConfig.DHCP = "ipv4"; # Fixed address inside the delegated /64: the IPv6 pinhole target. address = [ "${clientV6}/64" ]; }; networking.firewall.enable = false; # Every listener echoes the peer address it sees and its own port. services.nginx = { enable = true; virtualHosts.ports = { listen = map (port: { addr = "0.0.0.0"; inherit port; }) [ 8080 8084 9000 9001 9002 8085 8086 8087 ] ++ map (port: { addr = "[::]"; inherit port; }) [ 8081 8082 8080 ]; locations."/".return = ''200 "$remote_addr $server_port"''; }; }; }; # A second LAN host, outside the DHCP pool. peer = { lib, ... }: { virtualisation.vlans = [ 2 ]; networking.useNetworkd = true; networking.useDHCP = false; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; systemd.network.networks."10-lan" = { matchConfig.Name = "eth1"; address = [ "${peerV4}/24" ]; gateway = [ "192.168.10.1" ]; }; networking.firewall.enable = false; }; }; testScript = '' start_all() upstream.wait_for_unit("dnsmasq.service") upstream.wait_for_unit("nginx.service") with subtest("router obtains its IPv4 uplink and brings up services"): router.wait_for_unit("network-online.target") ${indent 4 uplinkWait} router.wait_for_unit("nftables.service") router.wait_for_unit("kea-dhcp4-server.service") router.wait_for_unit("unbound.service") with subtest("client gets address, gateway and DNS from the router"): client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q 192.168.10.") client.succeed("ip route show default | grep -q 'via 192.168.10.1'") client.succeed("resolvectl status eth1 | grep -q 192.168.10.1") with subtest("client resolves names through the router"): client.wait_until_succeeds("dig +short +time=2 +tries=1 @192.168.10.1 example.test | grep -qx ${ispV4}") with subtest("client reaches the 'internet' and is NATed"): out = client.wait_until_succeeds("curl -sf --max-time 5 --resolve example.test:80:${ispV4} http://example.test/") assert out.strip() == "${routerV4}", f"expected NATed source ${routerV4}, got {out!r}" ${lib.optionalString pppoe (indent 0 mssSubtest)} with subtest("WAN cannot initiate to the router or the LAN"): # Positive control: the probe tool works against a port that is open. upstream.succeed("nc -z -w 2 ${ispV4} 80") upstream.fail("nc -z -w 2 ${routerV4} 22") upstream.fail("nc -z -w 2 ${routerV4} 53") upstream.fail("ping -c 1 -W 2 192.168.10.1") upstream.fail("ping -c 1 -W 2 ${routerV4}") client_ip = client.succeed("ip -4 -o addr show dev eth1 | awk '{print $4}' | cut -d/ -f1").strip() upstream.fail(f"ping -c 1 -W 2 {client_ip}") with subtest("router gets an IPv6 address, the delegated /64 and a default route"): router.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:1::100") router.wait_until_succeeds("ip -6 addr show dev eth2 scope global | grep -q 2001:db8:2:") router.wait_until_succeeds("ip -6 route show default | grep -q 'via fe80:.* dev eth1'") # The WAN prefix is not autoconfigured, so DHCPv6 is the only global address there. assert router.succeed("ip -6 -o addr show dev eth1 scope global | wc -l").strip() == "1" with subtest("client autoconfigures a global address from the delegated prefix"): client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:") client.wait_until_succeeds("ip -6 route show default | grep -q 'dev eth1'") client_v6 = client.succeed("ip -6 -o addr show dev eth1 scope global | awk '{print $4}' | cut -d/ -f1 | grep -v ::1234 | head -1").strip() with subtest("IPv6 is routed without NAT"): client.wait_until_succeeds("ping -6 -c 1 -W 2 2001:db8:1::1") out = client.succeed("curl -sf -g --max-time 5 -H 'Host: example.test' 'http://[2001:db8:1::1]/'") assert out.strip() == client_v6, f"expected the client's own address {client_v6}, got {out!r}" with subtest("WAN cannot initiate IPv6 to the router or the LAN"): # Positive control: the ISP really has a route to the client, so a # failure below is down to the firewall. upstream.succeed(f"ip -6 route get {client_v6} | grep -q 'via 2001:db8:1::100'") upstream.fail(f"ping -6 -c 1 -W 2 {client_v6}") upstream.fail("ping -6 -c 1 -W 2 2001:db8:1::100") upstream.fail("nc -6 -z -w 2 2001:db8:1::100 22") with subtest("LAN can reach SSH and ping on the router"): client.succeed("nc -z -w 2 192.168.10.1 22") client.succeed("ping -c 1 -W 2 192.168.10.1") with subtest("port forward keeps the source address and honours toPort"): for _ in range(3): # Positive control: the listener answers on the LAN. peer.wait_until_succeeds("curl -sf --max-time 5 http://${clientV4}:8080/") out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8080/") assert out.strip() == "${ispV4} 8080", out out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8083/") assert out.strip() == "${ispV4} 8084", out # The translated port is not exposed directly. upstream.fail("curl -sf --max-time 3 http://${routerV4}:8084/") with subtest("port ranges open exactly their ports"): for port in (9000, 9001, 9002): out = upstream.succeed(f"curl -sf --max-time 5 http://${routerV4}:{port}/") assert out.strip() == f"${ispV4} {port}", out upstream.fail("curl -sf --max-time 3 http://${routerV4}:9003/") with subtest("source allow-list admits matching clients only"): out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8085/") assert out.strip() == "${ispV4} 8085", out upstream.fail("curl -sf --max-time 3 http://${routerV4}:8086/") with subtest("per-source rate limit drops excess new connections"): ok = 0 for _ in range(12): status, _out = upstream.execute("curl -sf --max-time 2 http://${routerV4}:8087/") ok += status == 0 assert 0 < ok < 12, f"expected some but not all connections to pass, got {ok}/12" with subtest("IPv6 reaches only the explicit pinholes"): out = upstream.succeed("curl -sf -g --max-time 5 'http://[${clientV6}]:8081/'") assert out.strip() == "2001:db8:1::1 8081", out # The listener exists, but there is no pinhole for it. client.succeed("nc -6 -z -w 2 ${clientV6} 8082") upstream.fail("curl -sf -g --max-time 3 'http://[${clientV6}]:8082/'") # Port 8080 is forwarded for IPv4 only. client.succeed("nc -6 -z -w 2 ${clientV6} 8080") upstream.fail("curl -sf -g --max-time 3 'http://[${clientV6}]:8080/'") with subtest("hairpin NAT works for forwards that enable it"): out = peer.succeed("curl -sf --max-time 5 http://${routerV4}:8080/") # Masqueraded: the server sees the router, not the peer. assert out.strip() == "192.168.10.1 8080", out # No hairpin on the range forward: the router does not translate it. peer.fail("curl -sf --max-time 3 http://${routerV4}:9001/") with subtest("logging rules leave a trace in the kernel log"): klog = router.succeed("journalctl -k --no-pager | grep 'router fwd#'") assert "accepted: " in klog, klog assert "rate-limited: " in klog, klog ''; }