nils/mg4
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 28 Wiki Activity

App 2.39.1, server 2.8.0: map roads from OpenFreeMap, on demand

Browse Source 
The public Overpass answered 429/502 and needs ~40 s a tile while the
car gives up after 8 s, so the server cancelled its queries and the
car's retries added load. Now the server fetches only the tiles a car
asks for, as OpenStreetMap vector tiles from OpenFreeMap (MAP_TILES_URL),
decodes the transportation layer with a small reader of its own, and
keeps each tile 30 days on a volume (MAP_CACHE_DIR). A fetch finishes
even after the car has stopped waiting; the car hears 503 with
Retry-After and gets the tile on its next try.

App: OpenFreeMap/OpenMapTiles attribution on the map, retry after 15 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LhzAVaUyqcw5PkGjz4mWW6
This commit is contained in:
nils
2026-10-26 23:54:30 +01:00
co-authored by Claude Opus 5.5
parent d9c7c5ea9a
commit 6be2ba5193
17 changed files with 891 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/maps_test.go
+246 -97
View File
@@ -12,50 +12,140 @@ import (
"strings"
"sync/atomic"
"testing"
"time"
)
// A tile in central Berlin.
var testTile = mapTile{Z: 14, X: 8802, Y: 5373}
// fakeOverpass answers every query with a primary road (with a gap in its geometry), a
// residential street with a needless middle point, and a footway the car does not draw.
func fakeOverpass(t *testing.T, calls *atomic.Int32, status int) *httptest.Server {
t.Helper()
s, w, n, e := tileBounds(testTile)
lat := func(f float64) float64 { return s + (n-s)*f }
lon := func(f float64) float64 { return w + (e-w)*f }
answer := fmt.Sprintf(`{"elements":[
{"type":"way","tags":{"highway":"primary"},"geometry":[
{"lat":%f,"lon":%f},{"lat":%f,"lon":%f},null,{"lat":%f,"lon":%f},{"lat":%f,"lon":%f}]},
{"type":"way","tags":{"highway":"residential"},"geometry":[
{"lat":%f,"lon":%f},{"lat":%f,"lon":%f},{"lat":%f,"lon":%f}]},
{"type":"way","tags":{"highway":"footway"},"geometry":[
{"lat":%f,"lon":%f},{"lat":%f,"lon":%f}]}]}`,
lat(0.1), lon(0.1), lat(0.2), lon(0.2), lat(0.6), lon(0.6), lat(0.7), lon(0.7),
lat(0.5), lon(0.1), lat(0.5), lon(0.5), lat(0.5), lon(0.9),
lat(0.3), lon(0.3), lat(0.4), lon(0.4))
return httptest.NewTLSServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
calls.Add(1)
if r.Method != http.MethodPost || !strings.Contains(r.UserAgent(), "nils-mg4-server") {
t.Errorf("request %s %q", r.Method, r.UserAgent())
}
_ = r.ParseForm()
if q := r.PostForm.Get("data"); !strings.Contains(q, "out geom(") || !strings.Contains(q, "residential") {
t.Errorf("query %q", q)
}
if status != http.StatusOK {
http.Error(rw, "busy", status)
return
}
_, _ = rw.Write([]byte(answer))
}))
// ---------- A minimal vector-tile encoder for the tests ----------
func pbVarint(v uint64) []byte {
var b []byte
for v >= 0x80 {
b = append(b, byte(v)|0x80)
v >>= 7
}
return append(b, byte(v))
}
func mapsAPI(overpass *httptest.Server) http.Handler {
func pbKey(num, wire int) []byte { return pbVarint(uint64(num<<3 | wire)) }
func pbBytes(num int, data []byte) []byte {
return append(append(pbKey(num, 2), pbVarint(uint64(len(data)))...), data...)
}
func pbUint(num int, v uint64) []byte { return append(pbKey(num, 0), pbVarint(v)...) }
func pbPacked(num int, vs []uint32) []byte {
var d []byte
for _, v := range vs {
d = append(d, pbVarint(uint64(v))...)
}
return pbBytes(num, d)
}
func zz(v int) uint32 { return uint32((v << 1) ^ (v >> 31)) }
// lineCmds encodes line strings as MoveTo/LineTo commands with relative moves.
func lineCmds(lines ...[][2]int) []uint32 {
var out []uint32
x, y := 0, 0
for _, l := range lines {
out = append(out, 1|1<<3, zz(l[0][0]-x), zz(l[0][1]-y))
x, y = l[0][0], l[0][1]
out = append(out, uint32(2|(len(l)-1)<<3))
for _, p := range l[1:] {
out = append(out, zz(p[0]-x), zz(p[1]-y))
x, y = p[0], p[1]
}
}
return out
}
type testFeature struct {
class int // index into the values below
geomType uint64
geometry []uint32
}
// testVectorTile has a water layer (ignored) and a transportation layer with the given extent.
func testVectorTile(extent uint64) []byte {
values := []string{"primary", "minor", "service", "motorway"}
features := []testFeature{
// A primary road with a needless middle point.
{0, 2, lineCmds([][2]int{{100, 100}, {200, 200}, {300, 300}})},
// A minor street in two pieces.
{1, 2, lineCmds([][2]int{{0, 500}, {400, 500}}, [][2]int{{600, 500}, {900, 520}})},
// A service road and a motorway as a polygon: neither is drawn.
{2, 2, lineCmds([][2]int{{10, 10}, {20, 20}})},
{3, 3, lineCmds([][2]int{{10, 10}, {20, 20}})},
}
var layer []byte
layer = append(layer, pbUint(15, 2)...)
layer = append(layer, pbBytes(1, []byte("transportation"))...)
for _, f := range features {
var feat []byte
feat = append(feat, pbPacked(2, []uint32{0, uint32(f.class)})...)
feat = append(feat, pbUint(3, f.geomType)...)
feat = append(feat, pbPacked(4, f.geometry)...)
layer = append(layer, pbBytes(2, feat)...)
}
layer = append(layer, pbBytes(3, []byte("class"))...)
for _, v := range values {
layer = append(layer, pbBytes(4, pbBytes(1, []byte(v)))...)
}
layer = append(layer, pbUint(5, extent)...)
water := append(pbBytes(1, []byte("water")), pbUint(5, 4096)...)
return append(pbBytes(3, water), pbBytes(3, layer)...)
}
// ---------- A fake tile host ----------
type fakeHost struct {
*httptest.Server
tiles atomic.Int32
status int
delay time.Duration
gzip bool
}
func newFakeHost(t *testing.T) *fakeHost {
t.Helper()
h := &fakeHost{status: http.StatusOK}
mux := http.NewServeMux()
mux.HandleFunc("GET /planet", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, `{"tilejson":"3.0.0","tiles":["%s/planet/build1/{z}/{x}/{y}.pbf"]}`, h.URL)
})
mux.HandleFunc("GET /planet/build1/{z}/{x}/{y}", func(w http.ResponseWriter, r *http.Request) {
h.tiles.Add(1)
if !strings.Contains(r.UserAgent(), "nils-mg4-server") {
t.Errorf("user agent %q", r.UserAgent())
}
time.Sleep(h.delay)
if h.status != http.StatusOK {
http.Error(w, "busy", h.status)
return
}
body := testVectorTile(4096)
if h.gzip {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
_, _ = zw.Write(body)
_ = zw.Close()
body = buf.Bytes()
}
_, _ = w.Write(body)
})
h.Server = httptest.NewTLSServer(mux)
return h
}
func mapsAPIWith(host *fakeHost, cacheDir string) http.Handler {
api, _ := newTestAPI()
api.Maps = NewMaps(overpass.URL, mapUserAgent)
api.Maps.Client = overpass.Client()
api.Maps.gap = 0
api.Maps = NewMaps(host.URL+"/planet", cacheDir, mapUserAgent)
api.Maps.Client = host.Client()
return api.Routes()
}
@@ -63,15 +153,8 @@ func tileBody(t mapTile) string {
return fmt.Sprintf(`{"z":%d,"x":%d,"y":%d}`, t.Z, t.X, t.Y)
}
func TestMapTileRoadsAreClassifiedSplitAndSimplified(t *testing.T) {
var calls atomic.Int32
overpass := fakeOverpass(t, &calls, http.StatusOK)
defer overpass.Close()
h := mapsAPI(overpass)
if rec := do(t, h, "POST", "/v1/map/roads", "", tileBody(testTile)); rec.Code != 401 {
t.Fatalf("without the car's token: %d", rec.Code)
}
rec := do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile))
func decodeAnswer(t *testing.T, rec *httptest.ResponseRecorder) roadTile {
t.Helper()
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body)
}
@@ -79,39 +162,100 @@ func TestMapTileRoadsAreClassifiedSplitAndSimplified(t *testing.T) {
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.Extent != 4096 || got.X != testTile.X || got.Y != testTile.Y {
return got
}
// ---------- Tests ----------
func TestVectorTileRoadsAreClassifiedSplitAndSimplified(t *testing.T) {
lines, err := decodeRoads(testVectorTile(4096))
if err != nil {
t.Fatal(err)
}
if len(lines) != 3 || lines[0].class != 1 || lines[1].class != 3 || lines[2].class != 3 {
t.Fatalf("lines %+v", lines)
}
if fmt.Sprint(lines[1].points) != "[[0 500] [400 500]]" || fmt.Sprint(lines[2].points) != "[[600 500] [900 520]]" {
t.Fatalf("pieces %v %v", lines[1].points, lines[2].points)
}
// Another extent is scaled to 4096.
half, err := decodeRoads(testVectorTile(8192))
if err != nil || fmt.Sprint(half[0].points[2]) != "[150 150]" {
t.Fatalf("scaled %v %v", half, err)
}
body, err := buildTile(testTile, testVectorTile(4096))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(body), `{"c":1,"p":[100,100,300,300]}`) {
t.Fatalf("the straight road keeps only its ends: %s", body)
}
}
func TestMalformedVectorTilesAreRefused(t *testing.T) {
good := testVectorTile(4096)
for name, raw := range map[string][]byte{
"cut short": good[:len(good)-7],
"bad wire type": {0x1f, 0x00},
"length too long": {0x1a, 0x7f, 0x01},
} {
if _, err := decodeRoads(raw); err == nil {
t.Errorf("%s: accepted", name)
}
}
if lines, err := decodeRoads(nil); err != nil || len(lines) != 0 {
t.Fatalf("an empty tile is an empty map: %v %v", lines, err)
}
}
func TestMapTilesAreFetchedOnceAndKeptOnDisk(t *testing.T) {
host := newFakeHost(t)
host.gzip = true // some hosts gzip whatever was asked
defer host.Close()
dir := t.TempDir()
h := mapsAPIWith(host, dir)
if rec := do(t, h, "POST", "/v1/map/roads", "", tileBody(testTile)); rec.Code != 401 {
t.Fatalf("without the car's token: %d", rec.Code)
}
got := decodeAnswer(t, do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile)))
if got.Extent != 4096 || got.X != testTile.X || got.Y != testTile.Y || len(got.Ways) != 3 {
t.Fatalf("tile %+v", got)
}
// The primary road in two pieces, the street straightened to its ends, no footway.
if len(got.Ways) != 3 || got.Ways[0].C != 1 || got.Ways[1].C != 1 || got.Ways[2].C != 3 {
t.Fatalf("ways %+v", got.Ways)
}
if len(got.Ways[2].P) != 4 {
t.Fatalf("the straight street keeps only its ends: %v", got.Ways[2].P)
}
// Tile-local: 0.1 of the way in is about 410 units from the west edge.
if x := got.Ways[0].P[0]; x < 400 || x > 420 {
t.Fatalf("x %d", x)
decodeAnswer(t, do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile)))
// A restart: a new server with the same directory asks the host nothing.
decodeAnswer(t, do(t, mapsAPIWith(host, dir), "POST", "/v1/map/roads", testToken, tileBody(testTile)))
if n := host.tiles.Load(); n != 1 {
t.Fatalf("tile fetched %d times", n)
}
}
// Cached: the second car asking costs Overpass nothing.
if rec := do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile)); rec.Code != 200 {
t.Fatalf("%d", rec.Code)
func TestASlowTileIsFinishedForTheCarsNextTry(t *testing.T) {
previous := mapWait
mapWait = 50 * time.Millisecond
defer func() { mapWait = previous }()
host := newFakeHost(t)
host.delay = 300 * time.Millisecond
defer host.Close()
h := mapsAPIWith(host, "")
rec := do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile))
if rec.Code != 503 || rec.Header().Get("Retry-After") == "" {
t.Fatalf("the car should hear 'try again': %d", rec.Code)
}
if calls.Load() != 1 {
t.Fatalf("overpass asked %d times", calls.Load())
time.Sleep(600 * time.Millisecond)
decodeAnswer(t, do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile)))
if n := host.tiles.Load(); n != 1 {
t.Fatalf("tile fetched %d times", n)
}
}
func TestMapTilesAreGzippedWhenTheCarAcceptsIt(t *testing.T) {
var calls atomic.Int32
overpass := fakeOverpass(t, &calls, http.StatusOK)
defer overpass.Close()
host := newFakeHost(t)
defer host.Close()
req := httptest.NewRequest("POST", "/v1/map/roads", strings.NewReader(tileBody(testTile)))
req.Header.Set("Authorization", "Bearer "+testToken)
req.Header.Set("Accept-Encoding", "gzip")
rec := httptest.NewRecorder()
mapsAPI(overpass).ServeHTTP(rec, req)
mapsAPIWith(host, "").ServeHTTP(rec, req)
if rec.Code != 200 || rec.Header().Get("Content-Encoding") != "gzip" {
t.Fatalf("%d %v", rec.Code, rec.Header())
}
@@ -126,84 +270,89 @@ func TestMapTilesAreGzippedWhenTheCarAcceptsIt(t *testing.T) {
}
func TestMapRequestsAreChecked(t *testing.T) {
var calls atomic.Int32
overpass := fakeOverpass(t, &calls, http.StatusOK)
defer overpass.Close()
h := mapsAPI(overpass)
host := newFakeHost(t)
defer host.Close()
h := mapsAPIWith(host, "")
for _, body := range []string{`{}`, `{"z":15,"x":1,"y":1}`, `{"z":14,"x":-1,"y":1}`,
`{"z":14,"x":1,"y":16384}`, `not json`} {
if rec := do(t, h, "POST", "/v1/map/roads", testToken, body); rec.Code != 400 {
t.Errorf("%s: %d", body, rec.Code)
}
}
if calls.Load() != 0 {
t.Fatal("a bad request reached Overpass")
if host.tiles.Load() != 0 {
t.Fatal("a bad request reached the tile host")
}
}
func TestABusyOverpassIsATemporaryFailureAndNotCached(t *testing.T) {
func TestAFailingHostIsATemporaryFailureAndNotCached(t *testing.T) {
var logged bytes.Buffer
previous := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(&logged, nil)))
defer slog.SetDefault(previous)
var calls atomic.Int32
overpass := fakeOverpass(t, &calls, http.StatusTooManyRequests)
defer overpass.Close()
h := mapsAPI(overpass)
host := newFakeHost(t)
host.status = http.StatusBadGateway
defer host.Close()
h := mapsAPIWith(host, t.TempDir())
for i := 0; i < 2; i++ {
if rec := do(t, h, "POST", "/v1/map/roads", testToken, tileBody(testTile)); rec.Code != 503 {
t.Fatalf("%d", rec.Code)
}
}
if calls.Load() != 2 {
t.Fatalf("a failure was cached: %d calls", calls.Load())
if n := host.tiles.Load(); n != 2 {
t.Fatalf("a failure was cached: %d fetches", n)
}
// Where the car is stays out of the log.
out := logged.String()
if !strings.Contains(out, "overpass answered 429") || strings.Contains(out, "8802") ||
if !strings.Contains(out, "tile host answered 502") || strings.Contains(out, "8802") ||
strings.Contains(out, "5373") {
t.Fatalf("log:\n%s", out)
}
}
func TestMapsAreOffWithoutOverpass(t *testing.T) {
func TestATileIndexWithoutHttpsIsRefused(t *testing.T) {
index := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"tiles":["http://tiles.example.com/{z}/{x}/{y}.pbf"]}`))
}))
defer index.Close()
api, _ := newTestAPI()
api.Maps = NewMaps(index.URL, "", mapUserAgent)
api.Maps.Client = index.Client()
if rec := do(t, api.Routes(), "POST", "/v1/map/roads", testToken, tileBody(testTile)); rec.Code != 503 {
t.Fatalf("%d", rec.Code)
}
}
func TestMapsAreOffWithoutATileSource(t *testing.T) {
api, _ := newTestAPI()
if rec := do(t, api.Routes(), "POST", "/v1/map/roads", testToken, tileBody(testTile)); rec.Code != 404 {
t.Fatalf("%d", rec.Code)
}
}
func TestOverpassURLConfiguration(t *testing.T) {
func TestMapTilesURLConfiguration(t *testing.T) {
base := map[string]string{"INFLUX_URL": "http://influx", "INFLUX_TOKEN": "t", "INFLUX_ORG": "o",
"INGEST_TOKEN": "0123456789abcdef01234567"}
load := func(overpass string) (config, error) {
load := func(tiles string) (config, error) {
return loadConfig(func(k string) string {
if k == "OVERPASS_URL" {
return overpass
if k == "MAP_TILES_URL" {
return tiles
}
return base[k]
})
}
if c, err := load(""); err != nil || c.overpassURL != DefaultOverpassURL {
t.Fatalf("default: %q %v", c.overpassURL, err)
if c, err := load(""); err != nil || c.mapTilesURL != DefaultMapTilesURL {
t.Fatalf("default: %q %v", c.mapTilesURL, err)
}
if c, err := load("off"); err != nil || c.overpassURL != "" {
t.Fatalf("off: %q %v", c.overpassURL, err)
if c, err := load("off"); err != nil || c.mapTilesURL != "" {
t.Fatalf("off: %q %v", c.mapTilesURL, err)
}
if _, err := load("http://overpass.example.com/api/interpreter"); err == nil {
if _, err := load("http://tiles.example.com/planet"); err == nil {
t.Fatal("plain http accepted")
}
}
func TestTilePointsAndSimplification(t *testing.T) {
s, w, n, e := tileBounds(testTile)
if x, y := tilePoint(testTile, n, w); x != 0 || y != 0 {
t.Fatalf("north-west corner %d,%d", x, y)
}
if x, y := tilePoint(testTile, s, e); x != 4096 || y != 4096 {
t.Fatalf("south-east corner %d,%d", x, y)
}
func TestSimplification(t *testing.T) {
line := [][2]int{{0, 0}, {10, 1}, {20, 0}, {30, 40}}
got := simplify(line, 4)
if len(got) != 3 || got[1] != [2]int{20, 0} {