# Router on a stick: one NIC carries the untagged LAN and, as VLAN 7, the # PPPoE uplink, like a single-port box behind a VLAN-tagged ISP. IPv6 runs # inside the PPP session: the ISP sends RAs and delegates a /64 on the PPP link. # # upstream (ISP, PPPoE on vlan 7) -- switch -- router (eth1) -- client, peer # # The ISP and the LAN share one L2 segment in the test, which is what a real # trunk looks like to the router: tagged frames go to the ISP, untagged to the LAN. { self, pkgs }: let inherit (pkgs) lib; common = import ./common.nix; vlan = 7; ispV4 = "198.51.100.1"; routerV4 = "198.51.100.100"; lanHost = address: { lib, ... }: { virtualisation.vlans = [ 1 ]; networking.useNetworkd = true; networking.useDHCP = false; networking.firewall.enable = false; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; systemd.network.networks."10-lan" = { matchConfig.Name = "eth1"; networkConfig.DHCP = if address == null then "ipv4" else "no"; address = lib.optional (address != null) "${address}/24"; gateway = lib.optional (address != null) "192.168.10.1"; }; }; in pkgs.testers.runNixOSTest { name = "router-onestick"; defaults.imports = [ common.tools ]; interactive.defaults = common.interactiveDefaults; nodes = { upstream = { lib, pkgs, ... }: { virtualisation.vlans = [ 1 ]; networking.useNetworkd = true; networking.useDHCP = false; networking.firewall.enable = false; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; boot.kernelModules = [ "pppoe" "pppox" "ppp_generic" ]; systemd.network.netdevs."05-pppoe" = { netdevConfig = { Kind = "vlan"; Name = "eth1.${toString vlan}"; }; vlanConfig.Id = vlan; }; systemd.network.networks."10-trunk" = { matchConfig.Name = "eth1"; networkConfig = { VLAN = [ "eth1.${toString vlan}" ]; LinkLocalAddressing = "no"; }; }; systemd.network.networks."11-pppoe" = { matchConfig.Name = "eth1.${toString vlan}"; networkConfig.LinkLocalAddressing = "no"; }; # The server's global address, reachable through the session. systemd.network.networks."05-lo" = { matchConfig.Name = "lo"; address = [ "2001:db8:1::1/128" ]; }; # RAs and DHCPv6 prefix delegation on the (dynamic) PPP device. Kea # cannot serve a point-to-point link (it only sees the peer's address), # so a minimal responder stands in for the ISP's DHCPv6 server. The test # starts both once the session exists. environment.systemPackages = [ pkgs.radvd pkgs.python3 ]; boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1; environment.etc."dhcp6-pd.py".text = '' import socket, struct, subprocess, sys PREFIX = socket.inet_pton(socket.AF_INET6, "2001:db8:2::") DEV = sys.argv[1] ifindex = socket.if_nametoindex(DEV) sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.setsockopt(socket.SOL_SOCKET, 25, DEV.encode() + b"\0") # SO_BINDTODEVICE sock.bind(("::", 547)) group = socket.inet_pton(socket.AF_INET6, "ff02::1:2") sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group + struct.pack("@I", ifindex)) SERVER_ID = struct.pack("!HH", 3, 0) + b"\x00\x03\x00\x01" + bytes(6) def options(data): out, i = {}, 0 while i + 4 <= len(data): code, length = struct.unpack("!HH", data[i:i + 4]) out[code] = data[i + 4:i + 4 + length] i += 4 + length return out def opt(code, body): return struct.pack("!HH", code, len(body)) + body print("dhcp6-pd: listening on", DEV, flush=True) while True: data, addr = sock.recvfrom(2048) mtype, xid, opts = data[0], data[1:4], options(data[4:]) if mtype not in (1, 3) or 25 not in opts: # Solicit, Request with IA_PD continue iaid = opts[25][:4] prefix = opt(26, struct.pack("!IIB", 3000, 4000, 64) + PREFIX) ia_pd = opt(25, iaid + struct.pack("!II", 1800, 2700) + prefix) rapid = mtype == 1 and 14 in opts reply = bytes([7 if (mtype == 3 or rapid) else 2]) + xid reply += opt(1, opts[1]) + opt(2, SERVER_ID[4:]) + ia_pd if rapid: reply += opt(14, b"") sock.sendto(reply, addr) print("dhcp6-pd: answered", mtype, "from", addr, flush=True) ''; environment.etc = { "ppp/options".text = ""; "ppp/chap-secrets".text = '' "testuser" * "p4ssw0rd" * ''; "ppp/pppoe-server-options".text = '' require-chap +ipv6 noccp mtu 1492 mru 1492 ''; }; systemd.services.pppoe-server = { wantedBy = [ "multi-user.target" ]; wants = [ "sys-subsystem-net-devices-eth1.${toString vlan}.device" ]; after = [ "systemd-networkd.service" "sys-subsystem-net-devices-eth1.${toString vlan}.device" ]; serviceConfig = { ExecStart = lib.concatStringsSep " " [ "${pkgs.rp-pppoe}/bin/pppoe-server -F -I eth1.${toString vlan}" "-L ${ispV4} -R ${routerV4} -N 1" "-O /etc/ppp/pppoe-server-options -q ${pkgs.ppp}/bin/pppd" ]; Restart = "always"; }; }; services.nginx = { enable = true; virtualHosts."isp".locations."/".return = ''200 "$remote_addr"''; }; }; router = { lib, ... }: { imports = [ self.nixosModules.router ]; virtualisation.vlans = [ 1 ]; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; environment.etc."pppoe-test-password" = { text = "p4ssw0rd"; mode = "0600"; }; router = { enable = true; wan = { interface = "eth1"; ipv6 = { enable = true; transport = "ppp"; }; ipv4 = { type = "pppoe"; pppoe = { inherit vlan; username = "testuser"; passwordFile = "/etc/pppoe-test-password"; }; }; }; lans.main = { interface = "eth1"; internet = true; ipv4.address = "192.168.10.1"; ipv6.pd = true; ipv6.address = "fd2c:cdef:cde9::1/64"; dhcp.pool = { start = "192.168.10.100"; end = "192.168.10.199"; }; }; # Only the PPP device is an uplink, so this must work from the ISP # and must not fire for traffic arriving from the LAN. firewall = { nat = [ { type = "dnat"; from = "wan"; to = "main"; toAddress = "192.168.10.51"; protocol = "tcp"; ports = [ 8080 ]; } ]; rules = [ { from = "wan"; to = "main"; destination = "192.168.10.51"; protocol = "tcp"; ports = [ 8080 ]; natOnly = true; } ]; }; dns = { forwarders = [ ispV4 ]; domain = "lan.test"; hosts.nas = [ "192.168.10.20" ]; }; }; }; client = lanHost null; peer = { lib, ... }: { imports = [ (lanHost "192.168.10.51") ]; services.nginx = { enable = true; virtualHosts."peer" = { listen = [ { addr = "0.0.0.0"; port = 8080; } ]; locations."/".return = ''200 "$remote_addr"''; }; }; }; }; testScript = '' start_all() upstream.wait_for_unit("nginx.service") peer.wait_for_unit("nginx.service") with subtest("the PPPoE session comes up on the tagged VLAN of the shared link"): upstream.wait_for_unit("pppoe-server.service") router.wait_for_unit("router-pppoe.service") router.wait_until_succeeds("ip -4 addr show dev pppoe-wan | grep -q ${routerV4}") router.wait_until_succeeds("ip -4 route show default | grep -q 'dev pppoe-wan'") assert "vlan protocol 802.1Q id ${toString vlan}" in router.succeed("ip -d link show eth1.${toString vlan}") router.wait_for_unit("nftables.service") router.wait_for_unit("kea-dhcp4-server.service") with subtest("the untagged side of the same NIC is the LAN"): router.succeed("ip -4 addr show dev eth1 | grep -q 192.168.10.1/24") client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q 192.168.10.100") client.succeed("ip route show default | grep -q 'via 192.168.10.1'") with subtest("the LAN reaches the ISP through PPPoE and is NATed"): out = client.wait_until_succeeds("curl -sf --max-time 5 http://${ispV4}/") assert out.strip() == "${routerV4}", out with subtest("the ISP cannot reach the router or the LAN"): upstream.succeed("nc -z -w 2 ${ispV4} 80") upstream.fail("nc -z -w 2 ${routerV4} 22") upstream.fail("nc -z -w 2 ${routerV4} 53") upstream.fail("ping -c 1 -W 2 ${routerV4}") with subtest("a port forward works from the ISP"): upstream.succeed("ip route replace 192.168.10.0/24 via ${routerV4}") out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8080/") assert out.strip() == "${ispV4}", out with subtest("the shared NIC is not an uplink: LAN traffic never hits WAN rules"): # Positive control: the target answers directly. client.succeed("curl -sf --max-time 5 http://192.168.10.51:8080/") # Same port on the router's LAN address: no forward applies (and the # input chain would drop it anyway), so this must fail. client.fail("curl -sf --max-time 3 http://192.168.10.1:8080/") router.succeed("nft list ruleset | grep -q 'iifname \"pppoe-wan\"'") assert 'iifname { "eth1"' not in router.succeed("nft list ruleset"), "eth1 treated as an uplink" with subtest("IPv6 comes up inside the PPP session"): upstream.wait_until_succeeds("ip link show ppp0") upstream.wait_until_succeeds("ip -6 -o addr show dev ppp0 scope link | grep -q inet6") upstream.succeed("systemd-run --unit=dhcp6pd python3 /etc/dhcp6-pd.py ppp0") upstream.succeed("printf 'interface ppp0 {\\n AdvSendAdvert on;\\n AdvDefaultLifetime 1800;\\n};\\n' > /tmp/radvd.conf") upstream.succeed("systemd-run --unit=radvd6 radvd -n -C /tmp/radvd.conf") router.wait_until_succeeds("ip -6 route show default | grep -q 'dev pppoe-wan'") router.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:") # Nothing IPv6 on the physical link or the PPPoE VLAN. router.fail("ip -6 addr show dev eth1.${toString vlan} | grep -q inet6") ll = router.succeed("ip -6 -o addr show dev pppoe-wan scope link | awk '{print $4}' | cut -d/ -f1 | head -1").strip() assert ll.startswith("fe80:"), ll # The ISP routes the delegated prefix to the router's link-local address. upstream.succeed(f"ip -6 route replace 2001:db8:2::/48 via {ll} dev ppp0") with subtest("the client autoconfigures from the delegated prefix, without NAT"): client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:") client_v6 = client.succeed("ip -6 -o addr show dev eth1 scope global | awk '{print $4}' | cut -d/ -f1 | grep 2001:db8:2: | head -1").strip() client.wait_until_succeeds("ping -6 -c 1 -W 2 2001:db8:1::1") out = client.succeed("curl -sf -g --max-time 5 'http://[2001:db8:1::1]/'") assert out.strip() == client_v6, f"expected {client_v6}, got {out!r}" with subtest("IPv6 MSS is clamped to the PPP MTU"): router.succeed("tcpdump -nn -i pppoe-wan -c 1 -w /tmp/syn6.pcap 'ip6 and ip6[6] == 6 and ip6[53] & 0x12 == 0x02' >/dev/null 2>&1 &") router.wait_until_succeeds("pgrep tcpdump") router.sleep(1) client.succeed("curl -sf -g --max-time 5 'http://[2001:db8:1::1]/'") mss = router.wait_until_succeeds("tcpdump -nn -v -r /tmp/syn6.pcap 2>&1 | grep -o 'mss [0-9]*'") assert mss.strip() == "mss 1432", f"expected mss 1432, got {mss!r}" with subtest("the ISP cannot initiate IPv6 to the router or the LAN"): # Positive control: the ISP really has a route to the client. upstream.succeed(f"ip -6 route get {client_v6} | grep -q 'dev ppp0'") upstream.fail(f"ping -6 -c 1 -W 2 {client_v6}") upstream.fail(f"nc -6 -z -w 2 {client_v6} 22") upstream.fail(f"nc -6 -z -w 2 {ll}%ppp0 22") with subtest("the ULA is announced and the resolver answers on it"): router.succeed("ip -6 addr show dev eth1 | grep -q fd2c:cdef:cde9::1/64") client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q fd2c:cdef:cde9:") client.succeed("ping -6 -c 1 -W 2 fd2c:cdef:cde9::1") out = client.succeed("dig +short +time=2 +tries=1 @fd2c:cdef:cde9::1 nas.lan.test").strip() assert out == "192.168.10.20", out # The RA carries the ULA prefix (SLAAC, on-link) and the resolver as RDNSS. # A link bounce makes the client solicit one right away. router.succeed("(timeout 20 tcpdump -nn -vv -i eth1 -c 1 'icmp6 and ip6[40] == 134' > /tmp/ra.txt 2>&1 &)") router.wait_until_succeeds("pgrep tcpdump") router.sleep(1) client.succeed("ip link set eth1 down; ip link set eth1 up") ra = router.wait_until_succeeds("grep -q 'router lifetime' /tmp/ra.txt && cat /tmp/ra.txt") assert "fd2c:cdef:cde9::/64, Flags [onlink, auto]" in ra, ra assert "rdnss option" in ra and "fd2c:cdef:cde9::1" in ra, ra ''; }