nilsandClaude Opus 5.5
5ddff44a35
Kea skips interfaces without carrier when it opens its sockets and only looks again on a reload, so a LAN cable plugged in after Kea started got no DHCP until a restart. networkd-dispatcher now restarts Kea when a DHCP LAN gets carrier (a restart: a SIGHUP while Kea is starting ends it). The rule script is the same for every configuration and reads the interfaces from /etc/router/dhcp-interfaces, so the device never has to rebuild it (writeShellApplication's shellcheck could not be built offline). The web UI refused every login from a browser with "forbidden": under Referrer-Policy no-referrer browsers send "Origin: null" with form posts. Use http.CrossOriginProtection (Sec-Fetch-Site, then Origin) and a same-origin referrer policy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AeFqV9dZrK6nipHkPfyM4F
528 lines
20 KiB
Nix
528 lines
20 KiB
Nix
# Single WAN + single LAN: DHCP, DNS, NAT, SLAAC and the default firewall
|
|
# policy for both address families. `wan4` selects how the IPv4 uplink works:
|
|
#
|
|
# dhcp - DHCPv4 on the uplink (ISP 203.0.113.1)
|
|
# pppoe - PPPoE over VLAN 7 on the uplink (ISP 198.51.100.1, PPP MTU 1492)
|
|
#
|
|
# IPv6 is native and untagged on the uplink in both variants.
|
|
#
|
|
# upstream (ISP) -- vlan 1 -- router -- vlan 2 -- client
|
|
#
|
|
# The fake ISP mimics the real uplink: RAs carry only a default route (no
|
|
# prefix to autoconfigure, no DHCPv6 flags), and Kea hands out one address
|
|
# plus a single delegated /64 out of 2001:db8:2::/48.
|
|
{
|
|
self,
|
|
pkgs,
|
|
wan4 ? "dhcp",
|
|
}:
|
|
let
|
|
pppoe = wan4 == "pppoe";
|
|
ispV4 = if pppoe then "198.51.100.1" else "203.0.113.1";
|
|
routerV4 = if pppoe then "198.51.100.100" else "203.0.113.100";
|
|
pppoeVlan = 7;
|
|
clientV4 = "192.168.10.100";
|
|
clientV6 = "2001:db8:2::1234";
|
|
peerV4 = "192.168.10.51";
|
|
|
|
inherit (pkgs) lib;
|
|
# Indent every line but the first (the interpolation site supplies that one).
|
|
indent =
|
|
n: text:
|
|
lib.concatStringsSep "\n${lib.fixedWidthString n " " ""}" (
|
|
lib.splitString "\n" (lib.removeSuffix "\n" text)
|
|
);
|
|
|
|
uplinkWait =
|
|
if pppoe then
|
|
''
|
|
upstream.wait_for_unit("pppoe-server.service")
|
|
router.wait_for_unit("router-pppoe.service")
|
|
router.wait_until_succeeds("ip -4 addr show dev pppoe-wan | grep -q ${routerV4}")
|
|
router.wait_until_succeeds("ip -4 route show default | grep -q 'dev pppoe-wan'")
|
|
# The ISP's route to the LAN needs the session to exist first.
|
|
upstream.wait_until_succeeds("ip -4 addr show | grep -q ${ispV4}")
|
|
upstream.succeed("ip route replace 192.168.10.0/24 via ${routerV4}")
|
|
''
|
|
else
|
|
''
|
|
router.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q ${routerV4}")
|
|
'';
|
|
|
|
mssSubtest = ''
|
|
with subtest("PPP link has MTU 1492 and forwarded TCP MSS is clamped"):
|
|
assert "mtu 1492" in router.succeed("ip link show pppoe-wan"), "unexpected PPP MTU"
|
|
# Capture the SYN as it leaves the router; the client itself offers MSS 1460.
|
|
router.succeed("tcpdump -nn -i pppoe-wan -c 1 -w /tmp/syn.pcap 'tcp[tcpflags] & (tcp-syn|tcp-ack) = tcp-syn' >/dev/null 2>&1 &")
|
|
router.wait_until_succeeds("pgrep tcpdump")
|
|
router.sleep(1)
|
|
client.succeed("curl -sf --max-time 5 --resolve example.test:80:${ispV4} http://example.test/")
|
|
mss = router.wait_until_succeeds("tcpdump -nn -v -r /tmp/syn.pcap 2>&1 | grep -o 'mss [0-9]*'")
|
|
assert mss.strip() == "mss 1452", f"expected the SYN clamped to mss 1452, got {mss!r}"
|
|
'';
|
|
|
|
common = import ./common.nix;
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "router-${if pppoe then "pppoe" else "basic"}";
|
|
|
|
defaults.imports = [ common.tools ];
|
|
|
|
interactive.defaults = common.interactiveDefaults;
|
|
|
|
nodes = {
|
|
# Plays the ISP: hands out one WAN lease (v4 and v6) plus a delegated /64,
|
|
# answers DNS, and serves a page that echoes the client address it sees
|
|
# (to prove NAT happens for v4 and does not for v6).
|
|
upstream =
|
|
{ lib, pkgs, ... }:
|
|
lib.mkMerge [
|
|
{
|
|
virtualisation.vlans = [ 1 ];
|
|
networking.useNetworkd = true;
|
|
networking.useDHCP = false;
|
|
networking.firewall.enable = false;
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
|
|
|
|
systemd.network.networks."10-wan" = {
|
|
matchConfig.Name = "eth1";
|
|
address = [ "2001:db8:1::1/64" ] ++ lib.optional (!pppoe) "203.0.113.1/24";
|
|
# Routes back to the LAN via the router so that "unreachable" below is
|
|
# due to the firewall and not a missing route. (With PPPoE the IPv4
|
|
# route can only exist once the session is up; the test adds it.)
|
|
routes = [
|
|
{
|
|
Destination = "2001:db8:2::/48";
|
|
Gateway = "2001:db8:1::100";
|
|
}
|
|
]
|
|
++ lib.optional (!pppoe) {
|
|
Destination = "192.168.10.0/24";
|
|
Gateway = "203.0.113.100";
|
|
};
|
|
networkConfig = {
|
|
IPv6SendRA = true;
|
|
}
|
|
// lib.optionalAttrs pppoe { VLAN = [ "eth1.${toString pppoeVlan}" ]; };
|
|
ipv6SendRAConfig = {
|
|
Managed = false;
|
|
OtherInformation = false;
|
|
RouterLifetimeSec = 1800;
|
|
EmitDNS = false;
|
|
};
|
|
# On-link only: the router must not autoconfigure a second address.
|
|
ipv6Prefixes = [
|
|
{
|
|
Prefix = "2001:db8:1::/64";
|
|
OnLink = true;
|
|
AddressAutoconfiguration = false;
|
|
}
|
|
];
|
|
};
|
|
|
|
services.dnsmasq = {
|
|
enable = true;
|
|
settings = {
|
|
address = "/example.test/${ispV4}";
|
|
}
|
|
// (
|
|
if pppoe then
|
|
# The PPP interface only exists once a session is up.
|
|
{ bind-dynamic = true; }
|
|
else
|
|
{
|
|
interface = "eth1";
|
|
bind-interfaces = true;
|
|
dhcp-range = "203.0.113.100,203.0.113.100,255.255.255.0,1h";
|
|
dhcp-option = [ "option:router,203.0.113.1" ];
|
|
}
|
|
);
|
|
};
|
|
|
|
services.kea.dhcp6 = {
|
|
enable = true;
|
|
settings = {
|
|
interfaces-config = {
|
|
interfaces = [ "eth1" ];
|
|
re-detect = true;
|
|
};
|
|
lease-database = {
|
|
type = "memfile";
|
|
persist = false;
|
|
};
|
|
preferred-lifetime = 3000;
|
|
valid-lifetime = 4000;
|
|
subnet6 = [
|
|
{
|
|
id = 1;
|
|
subnet = "2001:db8:1::/64";
|
|
interface = "eth1";
|
|
pools = [ { pool = "2001:db8:1::100 - 2001:db8:1::100"; } ];
|
|
pd-pools = [
|
|
{
|
|
prefix = "2001:db8:2::";
|
|
prefix-len = 48;
|
|
delegated-len = 64;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
virtualHosts."example.test".locations."/".return = ''200 "$remote_addr"'';
|
|
};
|
|
}
|
|
(lib.mkIf pppoe {
|
|
boot.kernelModules = [
|
|
"pppoe"
|
|
"pppox"
|
|
"ppp_generic"
|
|
];
|
|
systemd.network.netdevs."05-pppoe" = {
|
|
netdevConfig = {
|
|
Kind = "vlan";
|
|
Name = "eth1.${toString pppoeVlan}";
|
|
};
|
|
vlanConfig.Id = pppoeVlan;
|
|
};
|
|
systemd.network.networks."11-pppoe" = {
|
|
matchConfig.Name = "eth1.${toString pppoeVlan}";
|
|
networkConfig.LinkLocalAddressing = "no";
|
|
};
|
|
|
|
environment.etc = {
|
|
"ppp/options".text = "";
|
|
"ppp/chap-secrets".text = ''
|
|
"testuser" * "p4ss\"w0rd" *
|
|
'';
|
|
"ppp/pppoe-server-options".text = ''
|
|
require-chap
|
|
noccp
|
|
mtu 1492
|
|
mru 1492
|
|
lcp-echo-interval 10
|
|
lcp-echo-failure 3
|
|
'';
|
|
};
|
|
|
|
# The ISP's access concentrator: one session, 198.51.100.1 <-> .100.
|
|
systemd.services.pppoe-server = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "sys-subsystem-net-devices-eth1.${toString pppoeVlan}.device" ];
|
|
after = [
|
|
"systemd-networkd.service"
|
|
"sys-subsystem-net-devices-eth1.${toString pppoeVlan}.device"
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${pkgs.rp-pppoe}/bin/pppoe-server -F -I eth1.${toString pppoeVlan}"
|
|
"-L 198.51.100.1 -R 198.51.100.100 -N 1"
|
|
"-O /etc/ppp/pppoe-server-options -q ${pkgs.ppp}/bin/pppd"
|
|
];
|
|
Restart = "always";
|
|
};
|
|
};
|
|
})
|
|
];
|
|
|
|
router = { lib, ... }: {
|
|
imports = [ self.nixosModules.router ];
|
|
virtualisation.vlans = [
|
|
1
|
|
2
|
|
];
|
|
|
|
environment.etc."pppoe-test-password" = {
|
|
text = "p4ss\"w0rd";
|
|
mode = "0600";
|
|
};
|
|
|
|
# The test driver preconfigures static addresses on every vlan interface.
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
|
|
networking.interfaces.eth2.ipv4.addresses = lib.mkForce [ ];
|
|
|
|
router = {
|
|
enable = true;
|
|
wan = {
|
|
interface = "eth1";
|
|
ipv6.enable = true;
|
|
ipv4 = {
|
|
type = wan4;
|
|
}
|
|
// lib.optionalAttrs pppoe {
|
|
pppoe = {
|
|
vlan = pppoeVlan;
|
|
username = "testuser";
|
|
passwordFile = "/etc/pppoe-test-password";
|
|
};
|
|
};
|
|
};
|
|
lans.main = {
|
|
interface = "eth2";
|
|
internet = true;
|
|
ipv4 = {
|
|
address = "192.168.10.1";
|
|
};
|
|
ipv6.pd = true;
|
|
dhcp.pool = {
|
|
start = "192.168.10.100";
|
|
end = "192.168.10.199";
|
|
};
|
|
};
|
|
dns.forwarders = [ ispV4 ];
|
|
firewall.rules =
|
|
let
|
|
fwd =
|
|
r:
|
|
{
|
|
from = "wan";
|
|
to = "main";
|
|
destination = clientV4;
|
|
protocol = "tcp";
|
|
dnat = true;
|
|
}
|
|
// r;
|
|
in
|
|
[
|
|
(fwd {
|
|
ports = [ 8080 ];
|
|
hairpin = true;
|
|
log = true;
|
|
})
|
|
(fwd {
|
|
ports = [ 8083 ];
|
|
toPort = 8084;
|
|
})
|
|
(fwd { ports = [ "9000-9002" ]; })
|
|
(fwd {
|
|
ports = [ 8085 ];
|
|
sources = [ "${ispV4}/32" ];
|
|
})
|
|
(fwd {
|
|
ports = [ 8086 ];
|
|
sources = [ "192.0.2.0/24" ];
|
|
})
|
|
(fwd {
|
|
ports = [ 8087 ];
|
|
rateLimit = "3/minute";
|
|
log = true;
|
|
})
|
|
{
|
|
from = "wan";
|
|
to = "main";
|
|
destination = clientV6;
|
|
protocol = "tcp";
|
|
ports = [ 8081 ];
|
|
}
|
|
];
|
|
};
|
|
|
|
# Unbound answers `.test` locally (RFC 6761) and would validate DNSSEC
|
|
# against a fake ISP resolver that cannot provide it; neither applies to
|
|
# a real forwarder, so relax both for the made-up test domain only.
|
|
services.unbound.settings.server = {
|
|
local-zone = [ "test. transparent" ];
|
|
domain-insecure = [ "example.test" ];
|
|
};
|
|
};
|
|
|
|
client =
|
|
{ lib, ... }:
|
|
{
|
|
virtualisation.vlans = [ 2 ];
|
|
networking.useNetworkd = true;
|
|
networking.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
|
|
systemd.network.networks."10-lan" = {
|
|
matchConfig.Name = "eth1";
|
|
networkConfig.DHCP = "ipv4";
|
|
# Fixed address inside the delegated /64: the IPv6 pinhole target.
|
|
address = [ "${clientV6}/64" ];
|
|
};
|
|
networking.firewall.enable = false;
|
|
|
|
# Every listener echoes the peer address it sees and its own port.
|
|
services.nginx = {
|
|
enable = true;
|
|
virtualHosts.ports = {
|
|
listen =
|
|
map
|
|
(port: {
|
|
addr = "0.0.0.0";
|
|
inherit port;
|
|
})
|
|
[
|
|
8080
|
|
8084
|
|
9000
|
|
9001
|
|
9002
|
|
8085
|
|
8086
|
|
8087
|
|
]
|
|
++
|
|
map
|
|
(port: {
|
|
addr = "[::]";
|
|
inherit port;
|
|
})
|
|
[
|
|
8081
|
|
8082
|
|
8080
|
|
];
|
|
locations."/".return = ''200 "$remote_addr $server_port"'';
|
|
};
|
|
};
|
|
};
|
|
|
|
# A second LAN host, outside the DHCP pool.
|
|
peer =
|
|
{ lib, ... }:
|
|
{
|
|
virtualisation.vlans = [ 2 ];
|
|
networking.useNetworkd = true;
|
|
networking.useDHCP = false;
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ];
|
|
systemd.network.networks."10-lan" = {
|
|
matchConfig.Name = "eth1";
|
|
address = [ "${peerV4}/24" ];
|
|
gateway = [ "192.168.10.1" ];
|
|
};
|
|
networking.firewall.enable = false;
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
start_all()
|
|
|
|
upstream.wait_for_unit("dnsmasq.service")
|
|
upstream.wait_for_unit("nginx.service")
|
|
|
|
with subtest("router obtains its IPv4 uplink and brings up services"):
|
|
router.wait_for_unit("network-online.target")
|
|
${indent 4 uplinkWait}
|
|
router.wait_for_unit("nftables.service")
|
|
router.wait_for_unit("kea-dhcp4-server.service")
|
|
router.wait_for_unit("unbound.service")
|
|
|
|
with subtest("client gets address, gateway and DNS from the router"):
|
|
client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q 192.168.10.")
|
|
client.succeed("ip route show default | grep -q 'via 192.168.10.1'")
|
|
client.succeed("resolvectl status eth1 | grep -q 192.168.10.1")
|
|
|
|
with subtest("client resolves names through the router"):
|
|
client.wait_until_succeeds("dig +short +time=2 +tries=1 @192.168.10.1 example.test | grep -qx ${ispV4}")
|
|
|
|
with subtest("client reaches the 'internet' and is NATed"):
|
|
out = client.wait_until_succeeds("curl -sf --max-time 5 --resolve example.test:80:${ispV4} http://example.test/")
|
|
assert out.strip() == "${routerV4}", f"expected NATed source ${routerV4}, got {out!r}"
|
|
|
|
${lib.optionalString pppoe (indent 0 mssSubtest)}
|
|
with subtest("WAN cannot initiate to the router or the LAN"):
|
|
# Positive control: the probe tool works against a port that is open.
|
|
upstream.succeed("nc -z -w 2 ${ispV4} 80")
|
|
upstream.fail("nc -z -w 2 ${routerV4} 22")
|
|
upstream.fail("nc -z -w 2 ${routerV4} 53")
|
|
upstream.fail("ping -c 1 -W 2 192.168.10.1")
|
|
upstream.fail("ping -c 1 -W 2 ${routerV4}")
|
|
client_ip = client.succeed("ip -4 -o addr show dev eth1 | awk '{print $4}' | cut -d/ -f1").strip()
|
|
upstream.fail(f"ping -c 1 -W 2 {client_ip}")
|
|
|
|
with subtest("router gets an IPv6 address, the delegated /64 and a default route"):
|
|
router.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:1::100")
|
|
router.wait_until_succeeds("ip -6 addr show dev eth2 scope global | grep -q 2001:db8:2:")
|
|
router.wait_until_succeeds("ip -6 route show default | grep -q 'via fe80:.* dev eth1'")
|
|
# The WAN prefix is not autoconfigured, so DHCPv6 is the only global address there.
|
|
assert router.succeed("ip -6 -o addr show dev eth1 scope global | wc -l").strip() == "1"
|
|
|
|
with subtest("client autoconfigures a global address from the delegated prefix"):
|
|
client.wait_until_succeeds("ip -6 addr show dev eth1 scope global | grep -q 2001:db8:2:")
|
|
client.wait_until_succeeds("ip -6 route show default | grep -q 'dev eth1'")
|
|
client_v6 = client.succeed("ip -6 -o addr show dev eth1 scope global | awk '{print $4}' | cut -d/ -f1 | grep -v ::1234 | head -1").strip()
|
|
|
|
with subtest("IPv6 is routed without NAT"):
|
|
client.wait_until_succeeds("ping -6 -c 1 -W 2 2001:db8:1::1")
|
|
out = client.succeed("curl -sf -g --max-time 5 -H 'Host: example.test' 'http://[2001:db8:1::1]/'")
|
|
assert out.strip() == client_v6, f"expected the client's own address {client_v6}, got {out!r}"
|
|
|
|
with subtest("WAN cannot initiate IPv6 to the router or the LAN"):
|
|
# Positive control: the ISP really has a route to the client, so a
|
|
# failure below is down to the firewall.
|
|
upstream.succeed(f"ip -6 route get {client_v6} | grep -q 'via 2001:db8:1::100'")
|
|
upstream.fail(f"ping -6 -c 1 -W 2 {client_v6}")
|
|
upstream.fail("ping -6 -c 1 -W 2 2001:db8:1::100")
|
|
upstream.fail("nc -6 -z -w 2 2001:db8:1::100 22")
|
|
|
|
with subtest("LAN can reach SSH and ping on the router"):
|
|
client.succeed("nc -z -w 2 192.168.10.1 22")
|
|
client.succeed("ping -c 1 -W 2 192.168.10.1")
|
|
|
|
with subtest("port forward keeps the source address and honours toPort"):
|
|
for _ in range(3):
|
|
# Positive control: the listener answers on the LAN.
|
|
peer.wait_until_succeeds("curl -sf --max-time 5 http://${clientV4}:8080/")
|
|
out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8080/")
|
|
assert out.strip() == "${ispV4} 8080", out
|
|
out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8083/")
|
|
assert out.strip() == "${ispV4} 8084", out
|
|
# The translated port is not exposed directly.
|
|
upstream.fail("curl -sf --max-time 3 http://${routerV4}:8084/")
|
|
|
|
with subtest("port ranges open exactly their ports"):
|
|
for port in (9000, 9001, 9002):
|
|
out = upstream.succeed(f"curl -sf --max-time 5 http://${routerV4}:{port}/")
|
|
assert out.strip() == f"${ispV4} {port}", out
|
|
upstream.fail("curl -sf --max-time 3 http://${routerV4}:9003/")
|
|
|
|
with subtest("source allow-list admits matching clients only"):
|
|
out = upstream.succeed("curl -sf --max-time 5 http://${routerV4}:8085/")
|
|
assert out.strip() == "${ispV4} 8085", out
|
|
upstream.fail("curl -sf --max-time 3 http://${routerV4}:8086/")
|
|
|
|
with subtest("per-source rate limit drops excess new connections"):
|
|
ok = 0
|
|
for _ in range(12):
|
|
status, _out = upstream.execute("curl -sf --max-time 2 http://${routerV4}:8087/")
|
|
ok += status == 0
|
|
assert 0 < ok < 12, f"expected some but not all connections to pass, got {ok}/12"
|
|
|
|
with subtest("IPv6 reaches only the explicit pinholes"):
|
|
out = upstream.succeed("curl -sf -g --max-time 5 'http://[${clientV6}]:8081/'")
|
|
assert out.strip() == "2001:db8:1::1 8081", out
|
|
# The listener exists, but there is no pinhole for it.
|
|
client.succeed("nc -6 -z -w 2 ${clientV6} 8082")
|
|
upstream.fail("curl -sf -g --max-time 3 'http://[${clientV6}]:8082/'")
|
|
# Port 8080 is forwarded for IPv4 only.
|
|
client.succeed("nc -6 -z -w 2 ${clientV6} 8080")
|
|
upstream.fail("curl -sf -g --max-time 3 'http://[${clientV6}]:8080/'")
|
|
|
|
with subtest("hairpin NAT works for forwards that enable it"):
|
|
out = peer.succeed("curl -sf --max-time 5 http://${routerV4}:8080/")
|
|
# Masqueraded: the server sees the router, not the peer.
|
|
assert out.strip() == "192.168.10.1 8080", out
|
|
# No hairpin on the range forward: the router does not translate it.
|
|
peer.fail("curl -sf --max-time 3 http://${routerV4}:9001/")
|
|
|
|
with subtest("logging rules leave a trace in the kernel log"):
|
|
klog = router.succeed("journalctl -k --no-pager | grep 'router fwd#'")
|
|
assert "accepted: " in klog, klog
|
|
assert "rate-limited: " in klog, klog
|
|
|
|
with subtest("a LAN cable plugged in after Kea started still gets DHCP"):
|
|
# Like booting with nothing plugged in: Kea starts while the LAN NIC
|
|
# (eth2, QEMU netdev vlan2) has no carrier and skips it.
|
|
router.send_monitor_command("set_link vlan2 off")
|
|
router.wait_until_succeeds("grep -qx 0 /sys/class/net/eth2/carrier")
|
|
router.succeed("systemctl restart kea-dhcp4-server.service")
|
|
router.wait_until_succeeds("journalctl -u kea-dhcp4-server --since=-1min | grep -q 'eth2 is not running'")
|
|
router.send_monitor_command("set_link vlan2 on")
|
|
client.succeed("networkctl down eth1 && ip -4 addr flush dev eth1 && networkctl up eth1")
|
|
client.wait_until_succeeds("ip -4 addr show dev eth1 | grep -q 192.168.10.", timeout=60)
|
|
'';
|
|
}
|