2 Commits
Author SHA1 Message Date
nilsandClaude Opus 5.5 eebe840b6a App 2.41.0, server 2.9.0: recordings reach the server
A finished signal recording goes to an outbox and is sent gzipped with
the car's token (PUT /v1/logs/{name}); the service retries every 2
minutes while the car is offline. The server keeps the newest 200 on a
new volume (LOGS_DIR); reading takes a token of its own (LOGS_TOKEN).
fetch-logs in the dev shell fetches new ones into logs/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LhzAVaUyqcw5PkGjz4mWW6
2026-10-27 10:22:40 +01:00
nilsandClaude Opus 5.5 16e6f349da SPECS: app 2.40.4 published
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LhzAVaUyqcw5PkGjz4mWW6
2026-10-27 09:23:20 +01:00
20 changed files with 841 additions and 13 deletions
+4
View File
@@ -41,3 +41,7 @@ mise.local.toml
# CarPlay diagnostic exports: vendor software, never committed # CarPlay diagnostic exports: vendor software, never committed
/dump/ /dump/
# The car's own recordings (fetch-logs, or copied off a USB stick): its data, not the code's.
/logs/
/signal-recording-*.txt
+4 -2
View File
@@ -21,7 +21,8 @@ MG4 head unit (Android 9, AAOS, MT2712)
- **server/** — Go 1.27 API at `https://car.k8s.stinnesbeck.com`: stores telemetry, consumption - **server/** — Go 1.27 API at `https://car.k8s.stinnesbeck.com`: stores telemetry, consumption
and charging in InfluxDB, pairs cars by QR code, carries settings typed on a phone, relays and charging in InfluxDB, pairs cars by QR code, carries settings typed on a phone, relays
app releases from the private forge, and serves the roads for the Grid's map from app releases from the private forge, and serves the roads for the Grid's map from
OpenStreetMap vector tiles (OpenFreeMap), fetched per tile on demand and cached on a volume. It does **not** talk to ABRP. See `server/README.md`. OpenStreetMap vector tiles (OpenFreeMap), fetched per tile on demand and cached on a volume, and keeps the signal recordings the car sends
(read with their own token, `fetch-logs`). It does **not** talk to ABRP. See `server/README.md`.
- **k8s/** — plain Kubernetes manifests (InfluxDB, server, Grafana with three provisioned - **k8s/** — plain Kubernetes manifests (InfluxDB, server, Grafana with three provisioned
dashboards, Traefik routes). See `k8s/README.md`. dashboards, Traefik routes). See `k8s/README.md`.
- **EVHardware/** — vendored shared vehicle layer (PolyForm Noncommercial, see `LICENSE.md`). - **EVHardware/** — vendored shared vehicle layer (PolyForm Noncommercial, see `LICENSE.md`).
@@ -81,7 +82,7 @@ Repository: `ssh://git@git.stinnesbeck.com:2222/nils/mg4.git` (remote `origin`),
| Live data | on/off, ABRP API key and user token, upload frequency | | Live data | on/off, ABRP API key and user token, upload frequency |
| Server | URL, pairing by QR, ingest token, installation ID (read-only), config import | | Server | URL, pairing by QR, ingest token, installation ID (read-only), config import |
| Settings | drive mode (Eco, Normal, Sport, Snow, Custom; the lit one is what the car reports; changes only in P at a standstill, `DriveModeWrite`; "Set at start" sets the last confirmed choice again when the service starts with the head unit), full screen, voice key, Overview style with the accent colour beside it (offered only for styles drawn in it, `OverviewStyle.usesAccent`; Retro, Terminal, Grid (and Grid with map, sharing Grid's choice) and Visor add "Original" for their own palette, `hasOriginal`), tyre unit, battery and charging price (one card on wide screens), version, update check, install from USB, About | | Settings | drive mode (Eco, Normal, Sport, Snow, Custom; the lit one is what the car reports; changes only in P at a standstill, `DriveModeWrite`; "Set at start" sets the last confirmed choice again when the service starts with the head unit), full screen, voice key, Overview style with the accent colour beside it (offered only for styles drawn in it, `OverviewStyle.usesAccent`; Retro, Terminal, Grid (and Grid with map, sharing Grid's choice) and Visor add "Original" for their own palette, `hasOriginal`), tyre unit, battery and charging price (one card on wide screens), version, update check, install from USB, About |
| Logs | signal recorder (`SignalRecorder`/`SignalRecording`: every property and vendor getter via `VendorSurfaceProbe.capture`, again and again for 3 minutes, plus change events of the driver-assistance block it subscribes to, `EVHardware.listenToProperties` with `AssistBlock`, and steering-wheel key broadcasts; lists what changed, rarest first; saved to the app and a USB stick; how to find where the car publishes something), update log (every install step, with the platform's words), upload log | | Logs | signal recorder (`SignalRecorder`/`SignalRecording`: every property and vendor getter via `VendorSurfaceProbe.capture`, again and again for 3 minutes, plus change events of the driver-assistance block it subscribes to, `EVHardware.listenToProperties` with `AssistBlock`, and steering-wheel key broadcasts; lists what changed, rarest first; saved to the app, a USB stick and the server, through `LogOutbox`, which sends it again later when the car is offline; how to find where the car publishes something), update log (every install step, with the platform's words), upload log |
Key classes (`app/src/main/java/com/evsuite/abrp/`): `AbrpUploadService` (foreground service, Key classes (`app/src/main/java/com/evsuite/abrp/`): `AbrpUploadService` (foreground service,
15 s tick), `TelemetryUploader` (+ `TelemetryQueue`, persisted), `TelemetryPayload` (server 15 s tick), `TelemetryUploader` (+ `TelemetryQueue`, persisted), `TelemetryPayload` (server
@@ -110,6 +111,7 @@ nix develop # JDK 21, Android SDK, Go
./gradlew testStableDebugUnitTest testUnstableDebugUnitTest lintStableDebug ./gradlew testStableDebugUnitTest testUnstableDebugUnitTest lintStableDebug
build-car # build/car/NilsMG4-unstable-<version>.apk + .sha256 build-car # build/car/NilsMG4-unstable-<version>.apk + .sha256
publish-car # pre-release car-<version> on the forge (needs a token) publish-car # pre-release car-<version> on the forge (needs a token)
fetch-logs # the car's recordings from the server into logs/ (needs LOGS_TOKEN)
cd server && go vet ./... && go test ./... && govulncheck ./... cd server && go vet ./... && go test ./... && govulncheck ./...
``` ```
+1 -1
View File
@@ -95,7 +95,7 @@ with the server using a token of its own, obtained by pairing with a phone (belo
| **Live data** | Live data on/off, ABRP credentials for sending directly from the car, upload frequency | | **Live data** | Live data on/off, ABRP credentials for sending directly from the car, upload frequency |
| **Server** | Server URL, pairing by QR code, ingest token, installation ID, config import | | **Server** | Server URL, pairing by QR code, ingest token, installation ID, config import |
| **Settings** | Full screen, voice key, Overview style and accent colour, tyre pressure unit; drive mode (Eco, Normal, Sport, Snow, Custom: changes only in P at a standstill; the highlighted one is what the car reports; with "Set at start" the last confirmed choice is set again when the car starts, while it is still in P); battery (manual SoH, usable capacity) and charging price; entry on the phone; version, update check, install from USB stick, About | | **Settings** | Full screen, voice key, Overview style and accent colour, tyre pressure unit; drive mode (Eco, Normal, Sport, Snow, Custom: changes only in P at a standstill; the highlighted one is what the car reports; with "Set at start" the last confirmed choice is set again when the car starts, while it is still in P); battery (manual SoH, usable capacity) and charging price; entry on the phone; version, update check, install from USB stick, About |
| **Logs** | Signal recorder (records every value the head unit offers for 3 minutes, listens to the driver-assistance values for every change they report, and lists what changed, to find where the car publishes something; saved to a USB stick too), update log (every install step), upload log | | **Logs** | Signal recorder (records every value the head unit offers for 3 minutes, listens to the driver-assistance values for every change they report, and lists what changed, to find where the car publishes something; sent to your server, and saved to a USB stick too), update log (every install step), upload log |
- **Opening the app:** the MG4 launcher does not list third-party apps. **Double-press the - **Opening the app:** the MG4 launcher does not list third-party apps. **Double-press the
voice key** on the steering wheel and the app comes to the front (switch on the Settings voice key** on the steering wheel and the app comes to the front (switch on the Settings
+20
View File
@@ -140,6 +140,26 @@ tyre pressures, GPS (lat, lon, elevation, heading) and the is_* flags. It does *
confirmed car facts, app structure, UI conventions, build/test/ship, documents. It replaces confirmed car facts, app structure, UI conventions, build/test/ship, documents. It replaces
upstream's EVABRPUploader text. The test emulator is now in the flake: `nix run .#car-emulator` upstream's EVABRPUploader text. The test emulator is now in the flake: `nix run .#car-emulator`
(Android 9 AOSP at 1920×720, 160 dpi; the car build installs as the system user — checked). (Android 9 AOSP at 1920×720, 160 dpi; the car build installs as the system user — checked).
- 2026-10-27 — App 2.41.0, server 2.9.0: **recordings reach the server** (owner: "can we make
it so that we can upload the logs to an endpoint in the server, and you can get the logs from
there? that way i don't have to carry a usb stick"). A finished signal recording goes to
`files/log-outbox` (`LogOutbox`, at most 10 waiting) and is sent at once, gzipped, with the car's
token: `PUT /v1/logs/{name}`. Offline, 401 (not paired yet), 404 (a server without logs), 408,
429 or 5xx keep it, and the service's tick sends it again, at most every 2 minutes; any other
refusal drops it. The Logs tab says "Sent to the server." or how many wait and why. Server
(`logs.go`): `LOGS_DIR` on a new 1Gi volume `mg4-server-logs`, the newest 200 files and none
older than 180 days, names as the app's only, up to 2 MB sent and 8 MB unpacked, UTF-8 only; the
same name replaces itself. Reading (`GET /v1/logs`, `GET /v1/logs/{name}`) takes `LOGS_TOKEN`, a
new optional Secret value that must differ from `INGEST_TOKEN`; the car's token cannot read, the
read token cannot write. `fetch-logs` in the dev shell fetches the new ones into `logs/`
(gitignored, as are the recordings copied into the checkout's top folder), token from
`~/.config/nilsmg4/logs-token`. Tested: server unit tests (both tokens, refusals, pruning,
config); the real server code in a harness on this machine took the 08:52 recording gzipped as
the app sends it (204), refused the car's token for reading (401), and `fetch-logs` brought it
back byte for byte, once. The app's own sending could not be pointed at that harness (the app
takes https only); on the emulator the recording lands in the outbox and the Logs tab says it
waits. Not tried on the car.
- 2026-10-27 — App 2.40.4 published (`car-2.40.4`).
- 2026-10-27 — App 2.40.4: **the recorder's listening works on Android 9**. Both recordings - 2026-10-27 — App 2.40.4: **the recorder's listening works on Android 9**. Both recordings
from the car (08:49 and 08:52, 2.40.3) said "listening to the assistance properties: not from the car (08:49 and 08:52, 2.40.3) said "listening to the assistance properties: not
possible": `listenToProperties` looked only for `registerCallback`, Android 10's name; the car's possible": `listenToProperties` looked only for `registerCallback`, Android 10's name; the car's
+2 -2
View File
@@ -34,8 +34,8 @@ android {
applicationId "com.stinnesbeck.nilsmg4" applicationId "com.stinnesbeck.nilsmg4"
minSdk 27 minSdk 27
targetSdk 36 targetSdk 36
versionCode 149 versionCode 150
versionName "2.40.4" versionName "2.41.0"
// Nothing about the server ships in the APK: its URL and ingest token are entered in // Nothing about the server ships in the APK: its URL and ingest token are entered in
// the app's Server tab (ServerConfig), and the database is only reachable by the server. // the app's Server tab (ServerConfig), and the database is only reachable by the server.
@@ -223,6 +223,7 @@ public class AbrpUploadService extends Service {
consumptionTracker.sample(vehicle); consumptionTracker.sample(vehicle);
chargeSessionTracker.sample(vehicle); chargeSessionTracker.sample(vehicle);
consumptionCloudClient.syncIfDue(consumptionTracker, chargeSessionTracker, sampleMs); consumptionCloudClient.syncIfDue(consumptionTracker, chargeSessionTracker, sampleMs);
LogOutbox.sendIfDue(this); // recordings the car could not send when they ended
UploadSettings settings = UploadSettings.from(prefs); UploadSettings settings = UploadSettings.from(prefs);
// Live data goes to the server, to ABRP directly, or both; with neither set up there is // Live data goes to the server, to ABRP directly, or both; with neither set up there is
@@ -0,0 +1,156 @@
package com.evsuite.abrp;
import android.content.Context;
import android.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
/**
* Files from the Logs tab (signal recordings) on their way to the car server, so they reach the
* owner without a USB stick: kept in {@code files/log-outbox} until the server has them
* ({@code PUT /v1/logs/{name}}, gzip, the ingest token in the header), sent at once and, when
* that fails, again from the service's tick, at most every {@link #RETRY_MS}. At most
* {@link #MAX_FILES} wait; the oldest goes first. Names are the app's own and say nothing about
* where the car is.
*/
final class LogOutbox {
static final String DIR = "log-outbox";
static final int MAX_FILES = 10;
static final long RETRY_MS = 2 * 60_000L;
private static final String TAG = "EVABRP.LogOutbox";
/** As the server accepts them. */
private static final Pattern NAME = Pattern.compile("[a-z0-9][a-z0-9-]{0,79}\\.txt");
/** What to do with a file after the server's answer. */
enum Outcome { SENT, DROP, KEEP }
private static long lastFailureMs;
private static String lastResult;
private LogOutbox() {}
/** SENT for a 2xx; KEEP for what may pass later (no answer, 401 before pairing, 408, 429,
* 5xx, and a 404 from a server without logs yet); DROP for any other refusal. */
static Outcome outcome(int status) {
if (status >= 200 && status < 300) return Outcome.SENT;
if (status <= 0 || status == 401 || status == 404 || status == 408 || status == 429 || status >= 500) {
return Outcome.KEEP;
}
return Outcome.DROP;
}
static boolean validName(String name) {
return name != null && NAME.matcher(name).matches();
}
/** The names to remove so that at most [max] remain: the oldest (by name, which starts
* with what they are and ends in the time) beyond the newest [max]. */
static List<String> surplus(List<String> names, int max) {
List<String> sorted = new ArrayList<>(names);
Collections.sort(sorted);
return sorted.size() <= max ? Collections.emptyList() : sorted.subList(0, sorted.size() - max);
}
/** Puts [text] in the outbox as [name]; false when it could not be written. */
static synchronized boolean add(Context context, String name, List<String> text) {
if (!validName(name)) return false;
File dir = new File(context.getFilesDir(), DIR);
if (!dir.isDirectory() && !dir.mkdirs()) return false;
File tmp = new File(dir, name + ".tmp");
try (Writer out = new OutputStreamWriter(new FileOutputStream(tmp), StandardCharsets.UTF_8)) {
for (String line : text) out.write(line + "\n");
} catch (Exception e) {
tmp.delete();
return false;
}
if (!tmp.renameTo(new File(dir, name))) {
tmp.delete();
return false;
}
for (String old : surplus(waiting(dir), MAX_FILES)) new File(dir, old).delete();
lastFailureMs = 0; // something new: try at once
return true;
}
/** Sends what waits, unless the last attempt failed less than {@link #RETRY_MS} ago. Blocking:
* call it off the main thread. */
static void sendIfDue(Context context) {
synchronized (LogOutbox.class) {
if (lastFailureMs != 0 && System.currentTimeMillis() - lastFailureMs < RETRY_MS) return;
}
send(context);
}
/** Sends every waiting file, oldest first, until one fails. Blocking. */
static synchronized void send(Context context) {
File dir = new File(context.getFilesDir(), DIR);
List<String> names = waiting(dir);
if (names.isEmpty()) return;
ServerConfig config = ServerConfig.get(context);
if (!config.isConfigured()) {
lastResult = "no server set up";
lastFailureMs = System.currentTimeMillis();
return;
}
Collections.sort(names);
for (String name : names) {
File file = new File(dir, name);
int status;
try {
byte[] body = gzip(Files.readAllBytes(file.toPath()));
status = ServerHttp.send("PUT", config.url + "/v1/logs/" + name, "Bearer " + config.token,
"text/plain; charset=utf-8", "gzip", body).status;
} catch (Exception e) {
Log.w(TAG, "Log not sent: " + e.getClass().getSimpleName());
status = 0;
}
Outcome outcome = outcome(status);
if (outcome == Outcome.KEEP) {
lastResult = status == 0 ? "server not reached" : "server answered " + status;
lastFailureMs = System.currentTimeMillis();
return;
}
file.delete();
lastResult = outcome == Outcome.SENT ? "sent" : "refused by the server (" + status + ")";
Log.i(TAG, "Log " + name + ": " + lastResult);
}
lastFailureMs = 0;
}
/** For the Logs tab: whether the last file reached the server, or why not yet. */
static synchronized String status(Context context) {
int waiting = waiting(new File(context.getFilesDir(), DIR)).size();
if (waiting == 0) return lastResult == null ? null : "sent".equals(lastResult)
? context.getString(R.string.log_outbox_sent) : lastResult;
return context.getString(R.string.log_outbox_waiting, waiting, lastResult == null ? "…" : lastResult);
}
private static List<String> waiting(File dir) {
String[] names = dir.list();
List<String> out = new ArrayList<>();
if (names == null) return out;
for (String n : Arrays.asList(names)) if (validName(n)) out.add(n);
return out;
}
private static byte[] gzip(byte[] data) throws java.io.IOException {
ByteArrayOutputStream buf = new ByteArrayOutputStream(data.length / 4 + 64);
try (GZIPOutputStream z = new GZIPOutputStream(buf)) {
z.write(data);
}
return buf.toByteArray();
}
}
@@ -59,6 +59,12 @@ final class ServerHttp {
*/ */
static Response send(String method, String url, String authorization, String contentType, byte[] body) static Response send(String method, String url, String authorization, String contentType, byte[] body)
throws Exception { throws Exception {
return send(method, url, authorization, contentType, null, body);
}
/** As above, with a {@code Content-Encoding} ({@code "gzip"}) for the body, or null. */
static Response send(String method, String url, String authorization, String contentType,
String contentEncoding, byte[] body) throws Exception {
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
try { try {
connection.setRequestMethod(method); connection.setRequestMethod(method);
@@ -69,6 +75,7 @@ final class ServerHttp {
connection.setDoOutput(true); connection.setDoOutput(true);
connection.setFixedLengthStreamingMode(body.length); connection.setFixedLengthStreamingMode(body.length);
connection.setRequestProperty("Content-Type", contentType); connection.setRequestProperty("Content-Type", contentType);
if (contentEncoding != null) connection.setRequestProperty("Content-Encoding", contentEncoding);
try (OutputStream out = connection.getOutputStream()) { out.write(body); } try (OutputStream out = connection.getOutputStream()) { out.write(body); }
} }
int status = connection.getResponseCode(); int status = connection.getResponseCode();
@@ -39,8 +39,8 @@ import java.util.Map;
* A full capture is far heavier than the Overview's reading, so the next one waits at least * A full capture is far heavier than the Overview's reading, so the next one waits at least
* {@link #MIN_INTERVAL_MS} and at least as long as the last one took: the recorder uses at most * {@link #MIN_INTERVAL_MS} and at least as long as the last one took: the recorder uses at most
* half the time. It runs on its own thread, on after the app leaves the screen (the service * half the time. It runs on its own thread, on after the app leaves the screen (the service
* keeps the process), and stops by itself. At the end the recording goes to the app's storage * keeps the process), and stops by itself. At the end the recording goes to the app's storage,
* and to the app's folder on every USB stick plugged in. * to the app's folder on every USB stick plugged in, and to the car server ({@link LogOutbox}).
*/ */
final class SignalRecorder { final class SignalRecorder {
@@ -181,6 +181,12 @@ final class SignalRecorder {
// A volume that cannot say what it is gets nothing. // A volume that cannot say what it is gets nothing.
} }
} }
// And to the server, so it reaches the owner without a stick; the service sends it again
// later when the car is offline now.
if (LogOutbox.add(app, name, text)) {
LogOutbox.send(app);
places.add("for the server");
}
return places.isEmpty() ? "not saved" : "saved: " + String.join(", ", places); return places.isEmpty() ? "not saved" : "saved: " + String.join(", ", places);
} }
@@ -204,6 +210,8 @@ final class SignalRecorder {
} else { } else {
out.add(context.getString(R.string.signal_recorder_done, SignalRecording.time(startedMs), out.add(context.getString(R.string.signal_recorder_done, SignalRecording.time(startedMs),
recording.captures(), recording.events(), result == null ? "…" : result)); recording.captures(), recording.events(), result == null ? "…" : result));
String sent = LogOutbox.status(context);
if (sent != null) out.add(sent);
} }
// Steering-wheel keys and the like first: few, and each one a lead. // Steering-wheel keys and the like first: few, and each one a lead.
List<String> notes = recording.notes(); List<String> notes = recording.notes();
+3 -1
View File
@@ -223,11 +223,13 @@
<string name="drive_mode_unreadable">Not changed: speed or gear unreadable.</string> <string name="drive_mode_unreadable">Not changed: speed or gear unreadable.</string>
<string name="drive_mode_unknown">an unknown mode (%1$s)</string> <string name="drive_mode_unknown">an unknown mode (%1$s)</string>
<string name="label_signal_recorder">Signal recorder</string> <string name="label_signal_recorder">Signal recorder</string>
<string name="signal_recorder_hint">Records every value the head unit offers for 3 minutes, listens to the driver-assistance properties for every change they report, and notes the steering-wheel keys; then lists what changed, the rarest first. To find a signal: start it, note the times you switch the feature on and off or change it, and look for the values that changed then. With a USB stick plugged in, the whole recording is saved on it too (Android/data/…/files).</string> <string name="signal_recorder_hint">Records every value the head unit offers for 3 minutes, listens to the driver-assistance properties for every change they report, and notes the steering-wheel keys; then lists what changed, the rarest first. To find a signal: start it, note the times you switch the feature on and off or change it, and look for the values that changed then. The whole recording goes to your server (later, if the car is offline), and with a USB stick plugged in it is saved on it too (Android/data/…/files).</string>
<string name="btn_signal_record">Record 3 minutes</string> <string name="btn_signal_record">Record 3 minutes</string>
<string name="btn_signal_stop">Stop recording</string> <string name="btn_signal_stop">Stop recording</string>
<string name="signal_recorder_running">Recording: %1$d:%2$02d of 3:00, %3$d captures, %4$d change events heard. Keys reported, then assistance properties heard, then values changed so far, the rarest first:</string> <string name="signal_recorder_running">Recording: %1$d:%2$02d of 3:00, %3$d captures, %4$d change events heard. Keys reported, then assistance properties heard, then values changed so far, the rarest first:</string>
<string name="signal_recorder_done">Recording from %1$s: %2$d captures, %3$d change events heard, %4$s. Keys reported, then assistance properties heard, then values changed, the rarest first:</string> <string name="signal_recorder_done">Recording from %1$s: %2$d captures, %3$d change events heard, %4$s. Keys reported, then assistance properties heard, then values changed, the rarest first:</string>
<string name="log_outbox_sent">Sent to the server.</string>
<string name="log_outbox_waiting">Not on the server yet (%1$d waiting: %2$s); it is sent again every 2 minutes.</string>
<string name="call_log_empty">No upload attempted yet.</string> <string name="call_log_empty">No upload attempted yet.</string>
<string name="update_log_title">Updates, newest first:</string> <string name="update_log_title">Updates, newest first:</string>
<string name="btn_flush_now">Upload now</string> <string name="btn_flush_now">Upload now</string>
@@ -0,0 +1,45 @@
package com.evsuite.abrp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
public class LogOutboxTest {
@Test
public void whatTheServerSaysDecidesTheFile() {
assertEquals(LogOutbox.Outcome.SENT, LogOutbox.outcome(204));
assertEquals(LogOutbox.Outcome.SENT, LogOutbox.outcome(200));
// Worth another try: no answer, not paired yet, a server without logs yet, busy, broken.
for (int status : new int[] {0, 401, 404, 408, 429, 500, 502, 503}) {
assertEquals(String.valueOf(status), LogOutbox.Outcome.KEEP, LogOutbox.outcome(status));
}
// Refused for good: it would never pass.
for (int status : new int[] {400, 403, 413, 415}) {
assertEquals(String.valueOf(status), LogOutbox.Outcome.DROP, LogOutbox.outcome(status));
}
}
@Test
public void namesAreTheServersKind() {
assertTrue(LogOutbox.validName("signal-recording-20261027-085240.txt"));
assertFalse(LogOutbox.validName("signal-recording-20261027-085240.txt.tmp"));
assertFalse(LogOutbox.validName("../escape.txt"));
assertFalse(LogOutbox.validName("Upper.txt"));
assertFalse(LogOutbox.validName(null));
}
@Test
public void theOldestGoBeyondTheLimit() {
assertEquals(Arrays.asList("signal-recording-20261027-084921.txt"),
LogOutbox.surplus(Arrays.asList("signal-recording-20261027-085240.txt",
"signal-recording-20261027-084921.txt", "signal-recording-20261027-090000.txt"), 2));
assertEquals(Collections.emptyList(),
LogOutbox.surplus(Collections.singletonList("signal-recording-20261027-085240.txt"), 2));
}
}
+38 -1
View File
@@ -161,10 +161,46 @@
echo "Published $tag ($(basename "$apk") and .sha256) at $forge/$repo/releases/tag/$tag" echo "Published $tag ($(basename "$apk") and .sha256) at $forge/$repo/releases/tag/$tag"
''; '';
}; };
# Fetches the files the car sent from its Logs tab (signal recordings) from the car
# server into logs/ (gitignored), those not there yet. Needs the server's LOGS_TOKEN,
# from $LOGS_TOKEN or ~/.config/nilsmg4/logs-token; sent in a header, never printed.
fetchLogs = pkgs.writeShellApplication {
name = "fetch-logs";
runtimeInputs = [ pkgs.curl pkgs.jq pkgs.coreutils pkgs.git ];
text = ''
server=''${SERVER_URL:-https://car.k8s.stinnesbeck.com}
token=''${LOGS_TOKEN:-}
if [ -z "$token" ] && [ -f "$HOME/.config/nilsmg4/logs-token" ]; then
token=$(tr -d '[:space:]' < "$HOME/.config/nilsmg4/logs-token")
fi
if [ -z "$token" ]; then
echo "fetch-logs: no token. Put the server's LOGS_TOKEN in ~/.config/nilsmg4/logs-token (or \$LOGS_TOKEN)." >&2
exit 1
fi
root=$(git rev-parse --show-toplevel)
mkdir -p "$root/logs"
auth=(-H "Authorization: Bearer $token")
fetched=0
while IFS= read -r name; do
# The server only stores such names; anything else is not a path to write to.
if ! [[ "$name" =~ ^[a-z0-9][a-z0-9-]{0,79}\.txt$ ]]; then
echo "fetch-logs: skipping an unexpected name" >&2
continue
fi
[ -f "$root/logs/$name" ] && continue
curl -sSf "''${auth[@]}" -o "$root/logs/$name.part" "$server/v1/logs/$name"
mv "$root/logs/$name.part" "$root/logs/$name"
echo "logs/$name"
fetched=$((fetched + 1))
done < <(curl -sSf "''${auth[@]}" "$server/v1/logs" | jq -r '.logs[].name')
echo "fetch-logs: $fetched new"
'';
};
in in
{ {
default = pkgs.mkShell { default = pkgs.mkShell {
packages = [ jdk android.androidsdk buildCar publishCar pkgs.go pkgs.gopls pkgs.govulncheck pkgs.influxdb2-server pkgs.influxdb2-cli ]; packages = [ jdk android.androidsdk buildCar publishCar fetchLogs pkgs.go pkgs.gopls pkgs.govulncheck pkgs.influxdb2-server pkgs.influxdb2-cli ];
ANDROID_HOME = sdk; ANDROID_HOME = sdk;
ANDROID_SDK_ROOT = sdk; ANDROID_SDK_ROOT = sdk;
@@ -179,6 +215,7 @@
echo " ./gradlew assembleUnstableDebug # build that can update itself (OTA)" echo " ./gradlew assembleUnstableDebug # build that can update itself (OTA)"
echo " build-car # unstable build signed for the car" echo " build-car # unstable build signed for the car"
echo " publish-car # publish it for the in-app updater" echo " publish-car # publish it for the in-app updater"
echo " fetch-logs # the car's recordings from the server, into logs/"
''; '';
}; };
}); });
+11 -2
View File
@@ -7,7 +7,7 @@ is applied automatically. Apply them yourself.
|---|---| |---|---|
| `namespace.yaml` | Namespace `mg4` | | `namespace.yaml` | Namespace `mg4` |
| `influxdb.yaml` | InfluxDB 2.7.12 StatefulSet (10Gi PVC, default storage class), cluster-internal Service, NetworkPolicy admitting only the server and Grafana | | `influxdb.yaml` | InfluxDB 2.7.12 StatefulSet (10Gi PVC, default storage class), cluster-internal Service, NetworkPolicy admitting only the server and Grafana |
| `server.yaml` | Go server Deployment (1 replica, non-root, read-only root filesystem) and Service | | `server.yaml` | Go server Deployment (1 replica, non-root, read-only root filesystem), Service, and two 1Gi PVCs: the map tile cache and the logs the car sends |
| `grafana.yaml` | Grafana 13.2.3 Deployment (1Gi PVC), Service; datasource and dashboards provisioned | | `grafana.yaml` | Grafana 13.2.3 Deployment (1Gi PVC), Service; datasource and dashboards provisioned |
| `grafana/` | The provisioned InfluxDB datasource and the MG4 dashboards (Live, Consumption, Charging) | | `grafana/` | The provisioned InfluxDB datasource and the MG4 dashboards (Live, Consumption, Charging) |
| `ingress.yaml` | Traefik `IngressRoute`s for car.k8s.stinnesbeck.com and Grafana, TLS from Traefik's default certificate | | `ingress.yaml` | Traefik `IngressRoute`s for car.k8s.stinnesbeck.com and Grafana, TLS from Traefik's default certificate |
@@ -22,7 +22,8 @@ is applied automatically. Apply them yourself.
covered by Traefik's default certificate. covered by Traefik's default certificate.
2. **Create the Secrets.** `cp k8s/secrets/secrets.example.yaml k8s/secrets/secrets.yaml` (gitignored), replace 2. **Create the Secrets.** `cp k8s/secrets/secrets.example.yaml k8s/secrets/secrets.yaml` (gitignored), replace
every `CHANGE-ME` there (`openssl rand -hex 32`): `influxdb-auth`, `mg4-server` (ingest token, every `CHANGE-ME` there (`openssl rand -hex 32`): `influxdb-auth`, `mg4-server` (ingest token,
approval password for pairing, the forge token for app updates) and `grafana-admin`. approval password for pairing, the forge token for app updates, the token that reads the
car's logs) and `grafana-admin`.
Keep the template itself on placeholders only. Keep the template itself on placeholders only.
3. **Registry access.** The `mg4-server` package on git.stinnesbeck.com is public, so no pull 3. **Registry access.** The `mg4-server` package on git.stinnesbeck.com is public, so no pull
secret is needed. If you make it private, add an `imagePullSecrets` entry to `server.yaml`. secret is needed. If you make it private, add an `imagePullSecrets` entry to `server.yaml`.
@@ -54,6 +55,14 @@ the forge that can only **read** the `nils/mg4` repository, put it there, apply
restart the server. Without it, updates are off (the car's updater gets a 404). `FORGE_URL` restart the server. Without it, updates are off (the car's updater gets a 404). `FORGE_URL`
and `FORGE_REPO` are set in `server.yaml`. and `FORGE_REPO` are set in `server.yaml`.
## Logs from the car
The car sends its signal recordings to the server (`server/README.md`, "Logs from the car"),
which keeps them on the `mg4-server-logs` volume at `LOGS_DIR`. Reading them needs `LOGS_TOKEN`
from the `mg4-server` Secret: `openssl rand -hex 32`, put it there (it must differ from
`INGEST_TOKEN`), apply the Secret and restart the server. For `fetch-logs` on your machine, put
the same value in `~/.config/nilsmg4/logs-token`. Without it the car still sends; nobody can read.
## Agent access ## Agent access
[`agent-access/`](agent-access/README.md) creates a service account limited to the `mg4` [`agent-access/`](agent-access/README.md) creates a service account limited to the `mg4`
+3
View File
@@ -30,6 +30,9 @@ stringData:
# relays the releases to the car; k8s/README.md, "App updates"). Leave it empty to switch # relays the releases to the car; k8s/README.md, "App updates"). Leave it empty to switch
# updates off. # updates off.
FORGE_TOKEN: CHANGE-ME-read-only-forge-token FORGE_TOKEN: CHANGE-ME-read-only-forge-token
# Reads the files the car sends from its Logs tab (GET /v1/logs). At least 24 characters and
# not the INGEST_TOKEN. Leave it empty to switch reading off; the car can still send.
LOGS_TOKEN: CHANGE-ME-openssl-rand-hex-32
--- ---
apiVersion: v1 apiVersion: v1
kind: Secret kind: Secret
+26 -1
View File
@@ -16,6 +16,22 @@ spec:
requests: requests:
storage: 1Gi storage: 1Gi
--- ---
# The files the car sends from its Logs tab (signal recordings), read with LOGS_TOKEN. Unlike
# the map cache these are not fetched again, so they get a volume of their own.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mg4-server-logs
namespace: mg4
labels:
app.kubernetes.io/name: mg4-server
app.kubernetes.io/part-of: mg4
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 1Gi
---
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
@@ -73,12 +89,16 @@ spec:
value: https://tiles.openfreemap.org/planet value: https://tiles.openfreemap.org/planet
- name: MAP_CACHE_DIR - name: MAP_CACHE_DIR
value: /var/cache/mg4-maps value: /var/cache/mg4-maps
# Files from the car's Logs tab, kept on the volume below (the newest 200, at most
# 180 days). Reading them needs LOGS_TOKEN from the Secret; empty switches reading off.
- name: LOGS_DIR
value: /var/lib/mg4-logs
- name: INFLUX_TOKEN - name: INFLUX_TOKEN
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
name: influxdb-auth name: influxdb-auth
key: admin-token key: admin-token
# INGEST_TOKEN, APPROVAL_PASSWORD, FORGE_TOKEN. # INGEST_TOKEN, APPROVAL_PASSWORD, FORGE_TOKEN, LOGS_TOKEN.
envFrom: envFrom:
- secretRef: - secretRef:
name: mg4-server name: mg4-server
@@ -107,10 +127,15 @@ spec:
volumeMounts: volumeMounts:
- name: maps - name: maps
mountPath: /var/cache/mg4-maps mountPath: /var/cache/mg4-maps
- name: logs
mountPath: /var/lib/mg4-logs
volumes: volumes:
- name: maps - name: maps
persistentVolumeClaim: persistentVolumeClaim:
claimName: mg4-server-maps claimName: mg4-server-maps
- name: logs
persistentVolumeClaim:
claimName: mg4-server-logs
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
+23
View File
@@ -25,6 +25,8 @@ Environment only; the server refuses to start when a required value is missing.
| `FORGE_URL`, `FORGE_REPO`, `FORGE_TOKEN` | optional; app updates (below). The token only needs to read the repository | | `FORGE_URL`, `FORGE_REPO`, `FORGE_TOKEN` | optional; app updates (below). The token only needs to read the repository |
| `MAP_TILES_URL` | a TileJSON naming the vector tiles the map's roads come from; default OpenFreeMap's `https://tiles.openfreemap.org/planet`, `off` switches maps off, otherwise https only | | `MAP_TILES_URL` | a TileJSON naming the vector tiles the map's roads come from; default OpenFreeMap's `https://tiles.openfreemap.org/planet`, `off` switches maps off, otherwise https only |
| `MAP_CACHE_DIR` | where fetched map tiles are kept across restarts (the manifest mounts a volume there); empty keeps them in memory only | | `MAP_CACHE_DIR` | where fetched map tiles are kept across restarts (the manifest mounts a volume there); empty keeps them in memory only |
| `LOGS_DIR` | where the files the car sends from its Logs tab are kept (the manifest mounts a volume there); empty switches logs off |
| `LOGS_TOKEN` | optional, at least 24 characters and not the `INGEST_TOKEN`; reads those files (below). Empty: the car can send, nobody can read |
| `LISTEN_ADDR` | default `:8080` | | `LISTEN_ADDR` | default `:8080` |
## Endpoints ## Endpoints
@@ -43,6 +45,7 @@ with (403 otherwise); the shared token works for any.
- `PUT /v1/charging-sessions/{sessionId}` — a finished charging session (AC or DC, as far as the car was awake) with its curve. - `PUT /v1/charging-sessions/{sessionId}` — a finished charging session (AC or DC, as far as the car was awake) with its curve.
- `GET /v1/consumption/history/{installationId}` — the car reads its own totals back. - `GET /v1/consumption/history/{installationId}` — the car reads its own totals back.
- `GET /v1/health` — `{"status":"ok"}`. - `GET /v1/health` — `{"status":"ok"}`.
- `PUT /v1/logs/{name}` — a file from the car's Logs tab (below).
A malformed request is a 400 naming the field; a database failure is a 500 without detail. A malformed request is a 400 naming the field; a database failure is a 500 without detail.
@@ -79,6 +82,26 @@ and is kept, and the car hears 503 with `Retry-After: 10` and gets the tile on i
404 when maps are off. The log says `map tile fetched` with the size, never the tile. 404 when maps are off. The log says `map tile fetched` with the size, never the tile.
Map data © OpenStreetMap contributors (ODbL), tiles © OpenMapTiles, served by OpenFreeMap. Map data © OpenStreetMap contributors (ODbL), tiles © OpenMapTiles, served by OpenFreeMap.
## Logs from the car
So a signal recording reaches the owner without a USB stick. The car sends each one when it ends
(again later when it is offline) with its own token: `PUT /v1/logs/{name}`, plain UTF-8 text,
gzipped (`Content-Encoding: gzip`) or not; at most 2 MB as sent and 8 MB unpacked. The name is
the app's (`signal-recording-20261027-085240.txt`: lower-case letters, digits and dashes,
`.txt`); the same name again replaces the file, so a retry is harmless. 204 when stored. The
newest 200 files are kept, none older than 180 days.
Reading takes `LOGS_TOKEN`, which can do nothing else, and the car's token cannot read:
```sh
curl -H "Authorization: Bearer $LOGS_TOKEN" https://car.k8s.stinnesbeck.com/v1/logs # {logs: [{name, bytes, received}]}, newest first
curl -H "Authorization: Bearer $LOGS_TOKEN" https://car.k8s.stinnesbeck.com/v1/logs/<name> # the text
```
`fetch-logs` in the dev shell fetches the new ones into `logs/` (token from
`~/.config/nilsmg4/logs-token`). 404 when reading is off. The log says `log received` with the
name and size, never the contents.
## Settings from a phone ## Settings from a phone
The ABRP credentials are long to type on the car. With pairing on, the car (with its own token) The ABRP credentials are long to type on the car. With pairing on, the car (with its own token)
+4
View File
@@ -21,6 +21,7 @@ import (
// /v1/settings/…, /settings settings typed on a phone (handoff_http.go) // /v1/settings/…, /settings settings typed on a phone (handoff_http.go)
// GET /v1/app/… app updates from the forge (releases.go) // GET /v1/app/… app updates from the forge (releases.go)
// POST /v1/map/roads the roads of one map tile (maps.go) // POST /v1/map/roads the roads of one map tile (maps.go)
// /v1/logs/… files from the Logs tab, read with LOGS_TOKEN (logs.go)
// //
// A car authenticates with the shared INGEST_TOKEN or with its own token from pairing // A car authenticates with the shared INGEST_TOKEN or with its own token from pairing
// (pairing_http.go). // (pairing_http.go).
@@ -34,6 +35,8 @@ type API struct {
Releases *Releases Releases *Releases
// Maps serves the roads for the Overview's map from OpenStreetMap vector tiles (maps.go); nil is off. // Maps serves the roads for the Overview's map from OpenStreetMap vector tiles (maps.go); nil is off.
Maps *Maps Maps *Maps
// Logs keeps the files the car sends from its Logs tab (logs.go); nil is off.
Logs *Logs
// PublicURL is the server's address as a phone reaches it, for the pairing QR code; // PublicURL is the server's address as a phone reaches it, for the pairing QR code;
// derived from the request when empty. // derived from the request when empty.
PublicURL string PublicURL string
@@ -54,6 +57,7 @@ func (a *API) Routes() http.Handler {
a.handoffRoutes(mux) a.handoffRoutes(mux)
a.releaseRoutes(mux) a.releaseRoutes(mux)
a.mapRoutes(mux) a.mapRoutes(mux)
a.logRoutes(mux)
return mux return mux
} }
+248
View File
@@ -0,0 +1,248 @@
package main
import (
"bytes"
"compress/gzip"
"crypto/subtle"
"errors"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
)
// Logs keeps the text files the car sends from its Logs tab (signal recordings) on a volume,
// so they reach the owner without a USB stick. The car sends with its ingest token; reading
// needs LOGS_TOKEN, a token of its own, so neither the car's token can read them nor this one
// write anything.
//
// PUT /v1/logs/{name} one file from the car (ingest token); the same name again replaces it
// GET /v1/logs the files, newest first (LOGS_TOKEN)
// GET /v1/logs/{name} one file (LOGS_TOKEN)
//
// The newest maxLogFiles files younger than maxLogAge are kept. A file's contents never go to
// the server log, only its name and size.
type Logs struct {
Dir string
// ReadToken is LOGS_TOKEN; empty switches reading off (the car can still send).
ReadToken string
mu sync.Mutex
now func() time.Time
}
const (
maxLogWireBytes = 2 << 20 // as sent, gzip or not
maxLogBytes = 8 << 20 // as stored
maxLogFiles = 200
maxLogAge = 180 * 24 * time.Hour
)
// logName is what a file may be called: the app's own names, e.g. signal-recording-20261027-085240.txt.
var logName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,79}\.txt$`)
// NewLogs keeps the files in dir, which must exist.
func NewLogs(dir, readToken string) *Logs {
return &Logs{Dir: dir, ReadToken: readToken, now: time.Now}
}
func (a *API) logRoutes(mux *http.ServeMux) {
if a.Logs == nil {
return
}
mux.HandleFunc("PUT /v1/logs/{name}", a.putLog)
mux.HandleFunc("GET /v1/logs", a.Logs.list)
mux.HandleFunc("GET /v1/logs/{name}", a.Logs.get)
}
func (a *API) putLog(w http.ResponseWriter, r *http.Request) {
if _, ok := a.authenticate(r); !ok {
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
return
}
name := r.PathValue("name")
if !logName.MatchString(name) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "name must be lower-case letters, digits and dashes, ending in .txt"})
return
}
text, status, msg := readLogBody(r)
if status != 0 {
writeJSON(w, status, map[string]string{"error": msg})
return
}
if err := a.Logs.save(name, text); err != nil {
slog.Error("log not stored", "name", name, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal server error"})
return
}
slog.Info("log received", "name", name, "bytes", len(text))
w.WriteHeader(http.StatusNoContent)
}
// readLogBody is the request's text, un-gzipped when it says so; a status and message when it
// is too large, not gzip after all, or not UTF-8 text.
func readLogBody(r *http.Request) ([]byte, int, string) {
wire, err := io.ReadAll(io.LimitReader(r.Body, maxLogWireBytes+1))
if err != nil {
return nil, http.StatusBadRequest, "body could not be read"
}
if len(wire) > maxLogWireBytes {
return nil, http.StatusRequestEntityTooLarge, "body is too large"
}
text := wire
switch strings.ToLower(r.Header.Get("Content-Encoding")) {
case "", "identity":
case "gzip":
zr, err := gzip.NewReader(bytes.NewReader(wire))
if err != nil {
return nil, http.StatusBadRequest, "body is not gzip"
}
text, err = io.ReadAll(io.LimitReader(zr, maxLogBytes+1))
if err != nil {
return nil, http.StatusBadRequest, "body is not gzip"
}
default:
return nil, http.StatusUnsupportedMediaType, "only gzip or no encoding"
}
if len(text) > maxLogBytes {
return nil, http.StatusRequestEntityTooLarge, "body is too large"
}
if !utf8.Valid(text) {
return nil, http.StatusBadRequest, "body is not UTF-8 text"
}
return text, 0, ""
}
// save writes the file whole (a temporary file renamed into place), then prunes.
func (l *Logs) save(name string, text []byte) error {
l.mu.Lock()
defer l.mu.Unlock()
tmp, err := os.CreateTemp(l.Dir, ".incoming-*")
if err != nil {
return err
}
defer os.Remove(tmp.Name()) // gone after the rename; cleans up after a failure
if _, err := tmp.Write(text); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(tmp.Name(), filepath.Join(l.Dir, name)); err != nil {
return err
}
l.prune()
return nil
}
type logEntry struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
Received time.Time `json:"received"`
}
// entries are the stored files, newest first.
func (l *Logs) entries() ([]logEntry, error) {
dir, err := os.ReadDir(l.Dir)
if err != nil {
return nil, err
}
var out []logEntry
for _, e := range dir {
if !e.Type().IsRegular() || !logName.MatchString(e.Name()) {
continue
}
info, err := e.Info()
if err != nil {
continue
}
out = append(out, logEntry{Name: e.Name(), Bytes: info.Size(), Received: info.ModTime().UTC()})
}
sort.Slice(out, func(i, j int) bool {
if !out[i].Received.Equal(out[j].Received) {
return out[i].Received.After(out[j].Received)
}
return out[i].Name > out[j].Name
})
return out, nil
}
// prune removes all but the newest maxLogFiles and anything older than maxLogAge.
func (l *Logs) prune() {
entries, err := l.entries()
if err != nil {
return
}
cutoff := l.now().Add(-maxLogAge)
for i, e := range entries {
if i >= maxLogFiles || e.Received.Before(cutoff) {
if err := os.Remove(filepath.Join(l.Dir, e.Name)); err != nil && !errors.Is(err, os.ErrNotExist) {
slog.Warn("old log not removed", "name", e.Name, "error", err)
}
}
}
}
// mayRead accepts LOGS_TOKEN only, compared in constant time; main refuses a short one.
func (l *Logs) mayRead(w http.ResponseWriter, r *http.Request) bool {
if l.ReadToken == "" {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "reading logs is off (no LOGS_TOKEN)"})
return false
}
supplied, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
if !ok || subtle.ConstantTimeCompare([]byte(supplied), []byte(l.ReadToken)) != 1 {
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
return false
}
return true
}
func (l *Logs) list(w http.ResponseWriter, r *http.Request) {
if !l.mayRead(w, r) {
return
}
l.mu.Lock()
entries, err := l.entries()
l.mu.Unlock()
if err != nil {
slog.Error("logs not listed", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal server error"})
return
}
writeJSON(w, http.StatusOK, map[string]any{"logs": append([]logEntry{}, entries...)})
}
func (l *Logs) get(w http.ResponseWriter, r *http.Request) {
if !l.mayRead(w, r) {
return
}
name := r.PathValue("name")
if !logName.MatchString(name) {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no such log"})
return
}
l.mu.Lock()
text, err := os.ReadFile(filepath.Join(l.Dir, name))
l.mu.Unlock()
if errors.Is(err, os.ErrNotExist) {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no such log"})
return
}
if err != nil {
slog.Error("log not read", "name", name, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal server error"})
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(text)
}
+212
View File
@@ -0,0 +1,212 @@
package main
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
const testLogsToken = "logs-read-token-0123456789abcdef"
func newLogsAPI(t *testing.T) (*API, string) {
t.Helper()
api, _ := newTestAPI()
dir := t.TempDir()
api.Logs = NewLogs(dir, testLogsToken)
return api, dir
}
func gzipped(t *testing.T, text string) []byte {
t.Helper()
var b bytes.Buffer
zw := gzip.NewWriter(&b)
if _, err := zw.Write([]byte(text)); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func putLog(t *testing.T, h http.Handler, name, token string, body []byte, encoding string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest("PUT", "/v1/logs/"+name, bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
if encoding != "" {
req.Header.Set("Content-Encoding", encoding)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
func TestTheCarSendsALogAndTheOwnerReadsIt(t *testing.T) {
api, _ := newLogsAPI(t)
h := api.Routes()
const name = "signal-recording-20261027-085240.txt"
text := "Signal recording, 37 captures\nlistening to 108 assistance properties\n"
if rec := putLog(t, h, name, testToken, gzipped(t, text), "gzip"); rec.Code != http.StatusNoContent {
t.Fatalf("put: %d %s", rec.Code, rec.Body)
}
rec := do(t, h, "GET", "/v1/logs", testLogsToken, "")
if rec.Code != 200 {
t.Fatalf("list: %d %s", rec.Code, rec.Body)
}
var list struct {
Logs []logEntry `json:"logs"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil || len(list.Logs) != 1 || list.Logs[0].Name != name ||
list.Logs[0].Bytes != int64(len(text)) {
t.Fatalf("list: %v %s", err, rec.Body)
}
rec = do(t, h, "GET", "/v1/logs/"+name, testLogsToken, "")
if rec.Code != 200 || rec.Body.String() != text || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
t.Fatalf("get: %d %q", rec.Code, rec.Body)
}
// Sent again (a retry), it replaces itself.
if rec := putLog(t, h, name, testToken, []byte("again\n"), ""); rec.Code != http.StatusNoContent {
t.Fatalf("put again: %d", rec.Code)
}
if rec := do(t, h, "GET", "/v1/logs/"+name, testLogsToken, ""); rec.Body.String() != "again\n" {
t.Fatalf("replaced: %q", rec.Body)
}
}
func TestLogTokensDoNotCross(t *testing.T) {
api, _ := newLogsAPI(t)
h := api.Routes()
name := "signal-recording-1.txt"
if rec := putLog(t, h, name, testLogsToken, []byte("x"), ""); rec.Code != http.StatusUnauthorized {
t.Fatalf("the read token wrote: %d", rec.Code)
}
if rec := putLog(t, h, name, testToken, []byte("x"), ""); rec.Code != http.StatusNoContent {
t.Fatalf("put: %d", rec.Code)
}
for _, path := range []string{"/v1/logs", "/v1/logs/" + name} {
for _, token := range []string{"", testToken, testLogsToken + "x"} {
if rec := do(t, h, "GET", path, token, ""); rec.Code != http.StatusUnauthorized {
t.Fatalf("GET %s with %q: %d", path, token, rec.Code)
}
}
}
}
func TestReadingIsOffWithoutALogsToken(t *testing.T) {
api, _ := newLogsAPI(t)
api.Logs.ReadToken = ""
h := api.Routes()
if rec := putLog(t, h, "a.txt", testToken, []byte("x"), ""); rec.Code != http.StatusNoContent {
t.Fatalf("put: %d", rec.Code)
}
if rec := do(t, h, "GET", "/v1/logs", "", ""); rec.Code != http.StatusNotFound {
t.Fatalf("list: %d", rec.Code)
}
}
func TestBadLogsAreRefused(t *testing.T) {
api, dir := newLogsAPI(t)
h := api.Routes()
for _, c := range []struct {
name, encoding string
body []byte
want int
}{
{"..%2Fescape.txt", "", []byte("x"), http.StatusBadRequest},
{"Upper.txt", "", []byte("x"), http.StatusBadRequest},
{"no-suffix", "", []byte("x"), http.StatusBadRequest},
{".hidden.txt", "", []byte("x"), http.StatusBadRequest},
{"a.txt", "gzip", []byte("not gzip"), http.StatusBadRequest},
{"a.txt", "br", []byte("x"), http.StatusUnsupportedMediaType},
{"a.txt", "", []byte{0xff, 0xfe, 0x00}, http.StatusBadRequest},
{"a.txt", "", make([]byte, maxLogWireBytes+1), http.StatusRequestEntityTooLarge},
{"a.txt", "gzip", gzipped(t, strings.Repeat("a", maxLogBytes+1)), http.StatusRequestEntityTooLarge},
} {
if rec := putLog(t, h, c.name, testToken, c.body, c.encoding); rec.Code != c.want {
t.Errorf("%s %s: %d, want %d (%s)", c.name, c.encoding, rec.Code, c.want, rec.Body)
}
}
if files, _ := os.ReadDir(dir); len(files) != 0 {
t.Fatalf("stored: %v", files)
}
if rec := do(t, h, "GET", "/v1/logs/missing.txt", testLogsToken, ""); rec.Code != http.StatusNotFound {
t.Fatalf("missing: %d", rec.Code)
}
}
func TestOldAndSurplusLogsArePruned(t *testing.T) {
dir := t.TempDir()
l := NewLogs(dir, testLogsToken)
// The new file gets the real time as its own, so the others are placed before it.
now := time.Now()
old := filepath.Join(dir, "old.txt")
if err := os.WriteFile(old, []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(old, now.Add(-maxLogAge-time.Hour), now.Add(-maxLogAge-time.Hour)); err != nil {
t.Fatal(err)
}
for i := 0; i < maxLogFiles+5; i++ {
p := filepath.Join(dir, fmt.Sprintf("f-%03d.txt", i))
if err := os.WriteFile(p, []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
at := now.Add(-time.Duration(maxLogFiles+5-i) * time.Minute)
if err := os.Chtimes(p, at, at); err != nil {
t.Fatal(err)
}
}
if err := l.save("new.txt", []byte("y")); err != nil {
t.Fatal(err)
}
entries, _ := l.entries()
if len(entries) != maxLogFiles || entries[0].Name != "new.txt" {
t.Fatalf("%d kept, first %s", len(entries), entries[0].Name)
}
if _, err := os.Stat(old); !os.IsNotExist(err) {
t.Fatal("the old one is still there")
}
if _, err := os.Stat(filepath.Join(dir, "f-000.txt")); !os.IsNotExist(err) {
t.Fatal("the oldest surplus one is still there")
}
}
func TestLogsConfig(t *testing.T) {
base := map[string]string{"INFLUX_URL": "http://i", "INFLUX_TOKEN": "t", "INFLUX_ORG": "o", "INGEST_TOKEN": testToken}
env := func(extra map[string]string) func(string) string {
return func(k string) string {
if v, ok := extra[k]; ok {
return v
}
return base[k]
}
}
for _, c := range []struct {
extra map[string]string
ok bool
}{
{map[string]string{}, true},
{map[string]string{"LOGS_DIR": "/l"}, true},
{map[string]string{"LOGS_DIR": "/l", "LOGS_TOKEN": testLogsToken}, true},
{map[string]string{"LOGS_TOKEN": testLogsToken}, false},
{map[string]string{"LOGS_DIR": "/l", "LOGS_TOKEN": "short"}, false},
{map[string]string{"LOGS_DIR": "/l", "LOGS_TOKEN": testToken}, false},
} {
if _, err := loadConfig(env(c.extra)); (err == nil) != c.ok {
t.Errorf("%v: %v", c.extra, err)
}
}
}
+23 -1
View File
@@ -36,6 +36,8 @@ type config struct {
forgeToken string forgeToken string
mapTilesURL string mapTilesURL string
mapCacheDir string mapCacheDir string
logsDir string
logsToken string
} }
// loadConfig reads the environment and fails closed: a missing InfluxDB setting or a // loadConfig reads the environment and fails closed: a missing InfluxDB setting or a
@@ -55,6 +57,8 @@ func loadConfig(getenv func(string) string) (config, error) {
forgeToken: getenv("FORGE_TOKEN"), forgeToken: getenv("FORGE_TOKEN"),
mapTilesURL: getenv("MAP_TILES_URL"), mapTilesURL: getenv("MAP_TILES_URL"),
mapCacheDir: getenv("MAP_CACHE_DIR"), mapCacheDir: getenv("MAP_CACHE_DIR"),
logsDir: getenv("LOGS_DIR"),
logsToken: getenv("LOGS_TOKEN"),
} }
if c.listenAddr == "" { if c.listenAddr == "" {
c.listenAddr = ":8080" c.listenAddr = ":8080"
@@ -89,6 +93,17 @@ func loadConfig(getenv func(string) string) (config, error) {
return config{}, errors.New("MAP_TILES_URL must be an https URL or \"off\"") return config{}, errors.New("MAP_TILES_URL must be an https URL or \"off\"")
} }
} }
// Logs from the car are optional (LOGS_DIR); reading them needs a token of their own, as
// long as the ingest token, and never the ingest token itself.
if c.logsToken != "" && c.logsDir == "" {
return config{}, errors.New("LOGS_TOKEN is set but LOGS_DIR is not")
}
if c.logsToken != "" && len(c.logsToken) < minIngestTokenLength {
return config{}, fmt.Errorf("LOGS_TOKEN must be at least %d characters", minIngestTokenLength)
}
if c.logsToken != "" && c.logsToken == c.ingestToken {
return config{}, errors.New("LOGS_TOKEN must differ from INGEST_TOKEN")
}
if c.approvalPass != "" && len(c.approvalPass) < minApprovalPasswordLength { if c.approvalPass != "" && len(c.approvalPass) < minApprovalPasswordLength {
return config{}, fmt.Errorf("APPROVAL_PASSWORD must be at least %d characters", minApprovalPasswordLength) return config{}, fmt.Errorf("APPROVAL_PASSWORD must be at least %d characters", minApprovalPasswordLength)
} }
@@ -127,6 +142,13 @@ func main() {
} }
api.Maps = NewMaps(cfg.mapTilesURL, cfg.mapCacheDir, mapUserAgent) api.Maps = NewMaps(cfg.mapTilesURL, cfg.mapCacheDir, mapUserAgent)
} }
if cfg.logsDir != "" {
if err := os.MkdirAll(cfg.logsDir, 0o700); err != nil {
slog.Error("logs directory not usable", "error", err)
os.Exit(1)
}
api.Logs = NewLogs(cfg.logsDir, cfg.logsToken)
}
srv := &http.Server{ srv := &http.Server{
Addr: cfg.listenAddr, Addr: cfg.listenAddr,
@@ -142,7 +164,7 @@ func main() {
go func() { go func() {
slog.Info("listening", "addr", cfg.listenAddr, "bucket", cfg.influxBucket, slog.Info("listening", "addr", cfg.listenAddr, "bucket", cfg.influxBucket,
"pairing", pairing.Enabled(), "pairedDevices", len(devices), "appUpdates", releases.enabled(), "pairing", pairing.Enabled(), "pairedDevices", len(devices), "appUpdates", releases.enabled(),
"maps", api.Maps != nil) "maps", api.Maps != nil, "logs", api.Logs != nil, "logsReadable", api.Logs != nil && cfg.logsToken != "")
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
slog.Error("server failed", "error", err) slog.Error("server failed", "error", err)
os.Exit(1) os.Exit(1)