App 2.39.1, server 2.8.0: map roads from OpenFreeMap, on demand

The public Overpass answered 429/502 and needs ~40 s a tile while the car gives up after 8 s, so the server cancelled its queries and the car's retries added load. Now the server fetches only the tiles a car asks for, as OpenStreetMap vector tiles from OpenFreeMap (MAP_TILES_URL), decodes the transportation layer with a small reader of its own, and keeps each tile 30 days on a volume (MAP_CACHE_DIR). A fetch finishes even after the car has stopped waiting; the car hears 503 with Retry-After and gets the tile on its next try. App: OpenFreeMap/OpenMapTiles attribution on the map, retry after 15 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LhzAVaUyqcw5PkGjz4mWW6
2026-10-26 23:54:30 +01:00
parent d9c7c5ea9a
commit 6be2ba5193
17 changed files with 891 additions and 332 deletions
@@ -12,53 +12,69 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 )

-// Maps serves the roads around the car for the Overview's map (the "Grid with map" style),
-// taken from OpenStreetMap through an Overpass API and cached here, so the car only ever
-// talks to this server and asks Overpass nothing itself.
+// Maps serves the roads around the car for the Overview's map (the "Grid with map" style). It
+// fetches only the tiles a car asks for, as OpenStreetMap vector tiles (OpenMapTiles schema,
+// by default from OpenFreeMap), keeps their roads and stores them, so each tile is fetched
+// once and the car only ever talks to this server.
 //
 //	POST /v1/map/roads   {"z":14,"x":…,"y":…}
 //	                     → {"z","x","y","extent":4096,"ways":[{"c":class,"p":[x,y,x,y,…]}]}
 //
 // The car names a web-mercator tile at zoom 14 (about 1.5 km across in central Europe), not
-// its position, and the request goes in the body so the tile stays out of access logs. Points
-// are tile-local in 0..4096 (they may lie a little outside: a road is kept up to the first
-// point beyond the edge), simplified to about 1.5 m. Classes: 0 motorway and trunk, 1 primary
-// and secondary, 2 tertiary and unclassified, 3 residential and living streets.
+// its position, and in the body so the tile stays out of access logs. Points are tile-local
+// in 0..4096 (a little beyond at the edges), simplified to about 1.5 m. Classes: 0 motorway and
+// trunk, 1 primary and secondary, 2 tertiary, 3 minor streets (residential, unclassified,
+// living streets).
 //
-// Overpass is asked one tile at a time, at most once a second; a tile is kept for 30 days,
-// up to mapCacheTiles of them. The tile's coordinates never appear in the log.
+// A fetch runs to its end even when the car stops waiting (the car gives up after a few
+// seconds), so its next try finds the tile. Tiles are kept in memory (the last mapMemoryTiles)
+// and, with a cache directory, on disk for mapCacheAge. The tile's coordinates never appear
+// in the log.
 type Maps struct {
-	OverpassURL string
-	UserAgent   string
-	Client      *http.Client
+	// TileJSONURL names the vector tiles: a TileJSON document whose first "tiles" template is
+	// used (OpenFreeMap moves its template with every weekly build).
+	TileJSONURL string
+	// CacheDir keeps fetched tiles across restarts; empty keeps them in memory only.
+	CacheDir  string
+	UserAgent string
+	Client    *http.Client

-	mu    sync.Mutex
-	tiles map[mapTile]mapEntry
-
-	fetchMu   sync.Mutex // one Overpass request at a time
-	lastFetch time.Time
-	gap       time.Duration
+	mu       sync.Mutex
+	memory   map[mapTile]mapEntry
+	inflight map[mapTile]*mapFlight
+	template string
+	tmplAt   time.Time
+	// slots limits fetches running at once.
+	slots chan struct{}
 }

 const (
-	mapZoom          = 14
-	mapExtent        = 4096
-	mapCacheTiles    = 3000
-	mapCacheAge      = 30 * 24 * time.Hour
-	maxOverpassBytes = 32 << 20
+	mapZoom        = 14
+	mapExtent      = 4096
+	mapMemoryTiles = 300
+	mapCacheAge    = 30 * 24 * time.Hour
+	maxTileBytes   = 8 << 20
+	mapFetchTime   = 60 * time.Second
+	templateAge    = 6 * time.Hour
 	// The simplification tolerance in tile units (about 0.37 m each at 52° N).
 	mapTolerance = 4.0
-	// DefaultOverpassURL is the public instance of the Overpass API.
-	DefaultOverpassURL = "https://overpass-api.de/api/interpreter"
-	// mapUserAgent names this service to Overpass, as its usage policy asks.
-	mapUserAgent = "nils-mg4-server/2.7 (private car dashboard; roads for an in-car map)"
+	// DefaultMapTilesURL is OpenFreeMap's TileJSON: free OpenStreetMap vector tiles, no key.
+	DefaultMapTilesURL = "https://tiles.openfreemap.org/planet"
+	// mapUserAgent names this service to the tile host.
+	mapUserAgent = "nils-mg4-server/2.8 (private car dashboard; roads for an in-car map)"
 )

+// mapWait is how long a request waits for a fetch: less than the car's own 8 s.
+var mapWait = 6 * time.Second
+
 type mapTile struct{ Z, X, Y int }

 type mapEntry struct {
@@ -66,12 +82,19 @@ type mapEntry struct {
 	at   time.Time
 }

-// roadClasses maps OSM's highway values to the four classes the car draws.
+type mapFlight struct {
+	done chan struct{}
+	body []byte
+	err  error
+}
+
+// roadClasses maps the OpenMapTiles transportation classes to the four the car draws; the
+// rest (service, track, path, rail, ferry…) are left out.
 var roadClasses = map[string]int{
-	"motorway": 0, "motorway_link": 0, "trunk": 0, "trunk_link": 0,
-	"primary": 1, "primary_link": 1, "secondary": 1, "secondary_link": 1,
-	"tertiary": 2, "tertiary_link": 2, "unclassified": 2,
-	"residential": 3, "living_street": 3,
+	"motorway": 0, "trunk": 0,
+	"primary": 1, "secondary": 1,
+	"tertiary": 2,
+	"minor":    3,
 }

 type roadTile struct {
@@ -87,92 +110,227 @@ type road struct {
 	P []int `json:"p"`
 }

-func NewMaps(overpassURL, userAgent string) *Maps {
-	return &Maps{OverpassURL: overpassURL, UserAgent: userAgent, gap: time.Second}
+func NewMaps(tileJSONURL, cacheDir, userAgent string) *Maps {
+	return &Maps{TileJSONURL: tileJSONURL, CacheDir: cacheDir, UserAgent: userAgent}
 }

 func (m *Maps) client() *http.Client {
 	if m.Client != nil {
 		return m.Client
 	}
-	return &http.Client{Timeout: 60 * time.Second}
+	return &http.Client{Timeout: mapFetchTime}
 }

-// tileBounds is the tile's south, west, north and east edge in degrees.
-func tileBounds(t mapTile) (s, w, n, e float64) {
-	scale := math.Exp2(float64(t.Z))
-	lon := func(x float64) float64 { return x/scale*360 - 180 }
-	lat := func(y float64) float64 { return math.Atan(math.Sinh(math.Pi*(1-2*y/scale))) * 180 / math.Pi }
-	return lat(float64(t.Y + 1)), lon(float64(t.X)), lat(float64(t.Y)), lon(float64(t.X + 1))
+// Tile returns the tile's roads as JSON and whether it came from a cache. Waiting ends with
+// ctx; the fetch itself goes on and is kept.
+func (m *Maps) Tile(ctx context.Context, t mapTile) ([]byte, bool, error) {
+	if body, ok := m.cached(t); ok {
+		return body, true, nil
+	}
+	m.mu.Lock()
+	if m.inflight == nil {
+		m.inflight = map[mapTile]*mapFlight{}
+		m.slots = make(chan struct{}, 2)
+	}
+	f, running := m.inflight[t]
+	if !running {
+		f = &mapFlight{done: make(chan struct{})}
+		m.inflight[t] = f
+		go m.run(t, f)
+	}
+	m.mu.Unlock()
+	select {
+	case <-f.done:
+		return f.body, false, f.err
+	case <-ctx.Done():
+		return nil, false, errors.New("still fetching")
+	}
 }

-// tilePoint is a position in the tile's own units: 0..extent across, y downwards.
-func tilePoint(t mapTile, lat, lon float64) (int, int) {
-	scale := math.Exp2(float64(t.Z)) * mapExtent
-	x := (lon + 180) / 360 * scale
-	r := lat * math.Pi / 180
-	y := (1 - math.Log(math.Tan(r)+1/math.Cos(r))/math.Pi) / 2 * scale
-	return int(math.Round(x - float64(t.X)*mapExtent)), int(math.Round(y - float64(t.Y)*mapExtent))
+func (m *Maps) run(t mapTile, f *mapFlight) {
+	defer func() {
+		m.mu.Lock()
+		delete(m.inflight, t)
+		m.mu.Unlock()
+		close(f.done)
+	}()
+	m.slots <- struct{}{}
+	defer func() { <-m.slots }()
+	ctx, cancel := context.WithTimeout(context.Background(), mapFetchTime)
+	defer cancel()
+	raw, err := m.fetch(ctx, t)
+	if err == nil {
+		f.body, err = buildTile(t, raw)
+	}
+	if err != nil {
+		f.err = err
+		return
+	}
+	m.store(t, f.body)
+	slog.Info("map tile fetched", "ways", strings.Count(string(f.body), `"c":`), "bytes", len(f.body))
 }

-// overpassQuery asks for the drivable roads in the tile, with a small margin so that roads
-// crossing its edge reach a point beyond it.
-func overpassQuery(t mapTile) string {
-	s, w, n, e := tileBounds(t)
-	dy, dx := (n-s)*0.03, (e-w)*0.03
-	box := fmt.Sprintf("%.6f,%.6f,%.6f,%.6f", s-dy, w-dx, n+dy, e+dx)
-	return "[out:json][timeout:50];way[\"highway\"~\"^(motorway|motorway_link|trunk|trunk_link|primary|primary_link|" +
-		"secondary|secondary_link|tertiary|tertiary_link|unclassified|residential|living_street)$\"](" + box +
-		");out geom(" + box + ");"
+func (m *Maps) cachePath(t mapTile) string {
+	return filepath.Join(m.CacheDir, fmt.Sprintf("%d_%d_%d.json", t.Z, t.X, t.Y))
 }

-type overpassResult struct {
-	Elements []struct {
-		Type     string            `json:"type"`
-		Tags     map[string]string `json:"tags"`
-		Geometry []*struct {
-			Lat float64 `json:"lat"`
-			Lon float64 `json:"lon"`
-		} `json:"geometry"`
-	} `json:"elements"`
+func (m *Maps) cached(t mapTile) ([]byte, bool) {
+	m.mu.Lock()
+	e, ok := m.memory[t]
+	m.mu.Unlock()
+	if ok && time.Since(e.at) <= mapCacheAge {
+		return e.body, true
+	}
+	if m.CacheDir == "" {
+		return nil, false
+	}
+	info, err := os.Stat(m.cachePath(t))
+	if err != nil || time.Since(info.ModTime()) > mapCacheAge {
+		return nil, false
+	}
+	body, err := os.ReadFile(m.cachePath(t))
+	if err != nil || !json.Valid(body) {
+		return nil, false
+	}
+	m.remember(t, body, info.ModTime())
+	return body, true
 }

-// buildTile turns Overpass's answer into the tile the car reads. A missing point (outside
-// the clipped geometry) splits a road in two.
+func (m *Maps) store(t mapTile, body []byte) {
+	m.remember(t, body, time.Now())
+	if m.CacheDir == "" {
+		return
+	}
+	tmp := m.cachePath(t) + ".tmp"
+	if err := os.WriteFile(tmp, body, 0o600); err != nil {
+		slog.Warn("map tile not stored", "error", err)
+		return
+	}
+	if err := os.Rename(tmp, m.cachePath(t)); err != nil {
+		slog.Warn("map tile not stored", "error", err)
+	}
+}
+
+// remember keeps the tile in memory, dropping the oldest when full.
+func (m *Maps) remember(t mapTile, body []byte, at time.Time) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.memory == nil {
+		m.memory = map[mapTile]mapEntry{}
+	}
+	if _, ok := m.memory[t]; !ok && len(m.memory) >= mapMemoryTiles {
+		var oldest mapTile
+		var first time.Time
+		for k, e := range m.memory {
+			if first.IsZero() || e.at.Before(first) {
+				oldest, first = k, e.at
+			}
+		}
+		delete(m.memory, oldest)
+	}
+	m.memory[t] = mapEntry{body: body, at: at}
+}
+
+func (m *Maps) get(ctx context.Context, u string, limit int64) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", m.UserAgent)
+	resp, err := m.client().Do(req)
+	if err != nil {
+		// The URL holds the tile: name the failure, not the request.
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			return nil, errors.New("tile host timed out")
+		}
+		return nil, errors.New("tile host could not be reached")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
+		return nil, fmt.Errorf("tile host answered %d", resp.StatusCode)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
+	if err != nil {
+		return nil, errors.New("tile host answer could not be read")
+	}
+	if int64(len(body)) > limit {
+		return nil, errors.New("tile host answer is too large")
+	}
+	return body, nil
+}
+
+// tileTemplate is the current {z}/{x}/{y} URL from the TileJSON, looked up every few hours.
+func (m *Maps) tileTemplate(ctx context.Context) (string, error) {
+	m.mu.Lock()
+	tmpl, at := m.template, m.tmplAt
+	m.mu.Unlock()
+	if tmpl != "" && time.Since(at) < templateAge {
+		return tmpl, nil
+	}
+	body, err := m.get(ctx, m.TileJSONURL, 1<<20)
+	if err != nil {
+		return "", err
+	}
+	var doc struct {
+		Tiles []string `json:"tiles"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil || len(doc.Tiles) == 0 {
+		return "", errors.New("tile index has no tiles")
+	}
+	tmpl = doc.Tiles[0]
+	u, err := url.Parse(strings.NewReplacer("{z}", "0", "{x}", "0", "{y}", "0").Replace(tmpl))
+	if err != nil || u.Scheme != "https" || u.Host == "" ||
+		!strings.Contains(tmpl, "{z}") || !strings.Contains(tmpl, "{x}") || !strings.Contains(tmpl, "{y}") {
+		return "", errors.New("tile index names no usable https template")
+	}
+	m.mu.Lock()
+	m.template, m.tmplAt = tmpl, time.Now()
+	m.mu.Unlock()
+	return tmpl, nil
+}
+
+func (m *Maps) fetch(ctx context.Context, t mapTile) ([]byte, error) {
+	tmpl, err := m.tileTemplate(ctx)
+	if err != nil {
+		return nil, err
+	}
+	u := strings.NewReplacer("{z}", strconv.Itoa(t.Z), "{x}", strconv.Itoa(t.X), "{y}", strconv.Itoa(t.Y)).Replace(tmpl)
+	raw, err := m.get(ctx, u, maxTileBytes)
+	if err != nil {
+		return nil, err
+	}
+	// Some hosts send the tile gzipped whatever the request said.
+	if len(raw) > 2 && raw[0] == 0x1f && raw[1] == 0x8b {
+		zr, err := gzip.NewReader(bytes.NewReader(raw))
+		if err != nil {
+			return nil, errors.New("tile is not readable")
+		}
+		raw, err = io.ReadAll(io.LimitReader(zr, maxTileBytes+1))
+		if err != nil || len(raw) > maxTileBytes {
+			return nil, errors.New("tile is not readable")
+		}
+	}
+	return raw, nil
+}
+
+// buildTile keeps the roads of the vector tile's transportation layer, in the car's format.
 func buildTile(t mapTile, raw []byte) ([]byte, error) {
-	var res overpassResult
-	if err := json.Unmarshal(raw, &res); err != nil {
-		return nil, fmt.Errorf("overpass answer: %w", err)
+	lines, err := decodeRoads(raw)
+	if err != nil {
+		return nil, err
 	}
 	out := roadTile{Z: t.Z, X: t.X, Y: t.Y, Extent: mapExtent, Ways: []road{}}
-	for _, el := range res.Elements {
-		class, ok := roadClasses[el.Tags["highway"]]
-		if el.Type != "way" || !ok {
+	for _, l := range lines {
+		pts := simplify(l.points, mapTolerance)
+		if len(pts) < 2 {
 			continue
 		}
-		var run [][2]int
-		flush := func() {
-			if pts := simplify(run, mapTolerance); len(pts) >= 2 {
-				flat := make([]int, 0, len(pts)*2)
-				for _, p := range pts {
-					flat = append(flat, p[0], p[1])
-				}
-				out.Ways = append(out.Ways, road{C: class, P: flat})
-			}
-			run = nil
+		flat := make([]int, 0, len(pts)*2)
+		for _, p := range pts {
+			flat = append(flat, p[0], p[1])
 		}
-		for _, g := range el.Geometry {
-			if g == nil {
-				flush()
-				continue
-			}
-			x, y := tilePoint(t, g.Lat, g.Lon)
-			if n := len(run); n > 0 && run[n-1] == [2]int{x, y} {
-				continue
-			}
-			run = append(run, [2]int{x, y})
-		}
-		flush()
+		out.Ways = append(out.Ways, road{C: l.class, P: flat})
 	}
 	return json.Marshal(out)
 }
@@ -216,100 +374,11 @@ func segmentDistance(p, a, b [2]int) float64 {
 	if dx == 0 && dy == 0 {
 		return math.Hypot(px-ax, py-ay)
 	}
-	u := math.Max(0, math.Min(1, ((px-ax)*dx+(py-ay)*dy)/(dx*dx+dy*dy)))
+	u := ((px-ax)*dx + (py-ay)*dy) / (dx*dx + dy*dy)
+	u = max(0, min(1, u))
 	return math.Hypot(px-ax-u*dx, py-ay-u*dy)
 }

-// Tile returns the tile's roads as JSON, from the cache or from Overpass, and whether it
-// came from the cache.
-func (m *Maps) Tile(ctx context.Context, t mapTile) ([]byte, bool, error) {
-	if body, ok := m.cached(t); ok {
-		return body, true, nil
-	}
-	m.fetchMu.Lock()
-	defer m.fetchMu.Unlock()
-	// Another request may have fetched it while this one waited.
-	if body, ok := m.cached(t); ok {
-		return body, true, nil
-	}
-	if wait := m.gap - time.Since(m.lastFetch); wait > 0 {
-		select {
-		case <-time.After(wait):
-		case <-ctx.Done():
-			return nil, false, ctx.Err()
-		}
-	}
-	m.lastFetch = time.Now()
-	raw, err := m.fetch(ctx, t)
-	if err != nil {
-		return nil, false, err
-	}
-	body, err := buildTile(t, raw)
-	if err != nil {
-		return nil, false, err
-	}
-	m.store(t, body)
-	return body, false, nil
-}
-
-func (m *Maps) cached(t mapTile) ([]byte, bool) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	e, ok := m.tiles[t]
-	if !ok || time.Since(e.at) > mapCacheAge {
-		return nil, false
-	}
-	return e.body, true
-}
-
-// store keeps the tile, dropping the oldest when the cache is full.
-func (m *Maps) store(t mapTile, body []byte) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.tiles == nil {
-		m.tiles = map[mapTile]mapEntry{}
-	}
-	if len(m.tiles) >= mapCacheTiles {
-		var oldest mapTile
-		var at time.Time
-		for k, e := range m.tiles {
-			if at.IsZero() || e.at.Before(at) {
-				oldest, at = k, e.at
-			}
-		}
-		delete(m.tiles, oldest)
-	}
-	m.tiles[t] = mapEntry{body: body, at: time.Now()}
-}
-
-// fetch asks Overpass. The query goes in the body, so its errors never carry the tile.
-func (m *Maps) fetch(ctx context.Context, t mapTile) ([]byte, error) {
-	form := url.Values{"data": {overpassQuery(t)}}.Encode()
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, m.OverpassURL, strings.NewReader(form))
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	req.Header.Set("User-Agent", m.UserAgent)
-	resp, err := m.client().Do(req)
-	if err != nil {
-		return nil, errors.New("overpass could not be reached")
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
-		return nil, fmt.Errorf("overpass answered %d", resp.StatusCode)
-	}
-	raw, err := io.ReadAll(io.LimitReader(resp.Body, maxOverpassBytes+1))
-	if err != nil {
-		return nil, errors.New("overpass answer could not be read")
-	}
-	if len(raw) > maxOverpassBytes {
-		return nil, errors.New("overpass answer is too large")
-	}
-	return raw, nil
-}
-
 func (a *API) mapRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("POST /v1/map/roads", a.ingest(a.mapRoads))
 }
@@ -332,17 +401,16 @@ func (a *API) mapRoads(w http.ResponseWriter, r *http.Request, c caller, body []
 	if *req.X < 0 || *req.X >= limit || *req.Y < 0 || *req.Y >= limit {
 		return invalid("x and y must be within the zoom level")
 	}
-	t := mapTile{Z: *req.Z, X: *req.X, Y: *req.Y}
-	data, cached, err := a.Maps.Tile(r.Context(), t)
+	// The car waits a few seconds at most; the fetch goes on without it.
+	wait, cancel := context.WithTimeout(r.Context(), mapWait)
+	defer cancel()
+	data, _, err := a.Maps.Tile(wait, mapTile{Z: *req.Z, X: *req.X, Y: *req.Y})
 	if err != nil {
-		slog.Warn("map tile not available", "installation", who(c), "error", err)
-		writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "map data not available, try again later"})
+		slog.Warn("map tile not available yet", "installation", who(c), "error", err)
+		w.Header().Set("Retry-After", "10")
+		writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "map data not available yet, try again shortly"})
 		return nil
 	}
-	if !cached {
-		slog.Info("map tile fetched", "installation", who(c), "ways", strings.Count(string(data), `"c":`),
-			"bytes", len(data))
-	}
 	w.Header().Set("Content-Type", "application/json")
 	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
 		var buf bytes.Buffer