diff --git a/TODO.md b/TODO.md index f4cf6a5..9329bde 100644 --- a/TODO.md +++ b/TODO.md @@ -197,3 +197,21 @@ Only matters with very large portfolios (100+ symbols) hitting ZFIN_SERVER. Cache store reads entire files into memory (`readFileAlloc` with 50MB limit). For portfolios with 10+ years of daily candles, this could use significant memory. Keep current approach unless memory becomes a real problem. + +## Torn SRF files from server sync (recurring bug) + +**Status:** First-layer fix done — 2026-05-02. `syncFromServer` +(`src/service.zig`) now validates responses via +`cache.Store.looksCompleteSrf` before `writeRaw`. Torn HTTP bodies +(empty, missing `#!srfv1` header, or no trailing newline) are +rejected with a warn-level log and NOT written to cache. Next fetch +will try the provider fallback. + +**Remaining work (if it comes back):** + +- HTTP-level Content-Length validation in `src/net/http.zig` to fail + closer to the source rather than at the cache write. +- Read-path self-heal: on SRF parse failure during read, invalidate + the cache entry so a subsequent refresh can repair without user + intervention. + diff --git a/src/cache/store.zig b/src/cache/store.zig index 3d82557..c531e87 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -316,6 +316,33 @@ pub const Store = struct { self.writeRaw(symbol, data_type, negative_cache_content) catch {}; } + /// Validate that a byte buffer looks like a complete SRF file. + /// + /// A well-formed SRF file should: + /// - Be non-empty. + /// - Start with the `#!srfv1` version directive (the negative-cache + /// marker also starts this way, so a single check covers both). + /// - End with a newline. Every record and every directive is + /// newline-terminated; a completely-written file always has a + /// trailing `\n`. + /// + /// Torn HTTP body writes land short of the final newline. The FRDM + /// corruption on 2026-05-02 is the canonical example: the tail of + /// `candles_daily.srf` ended with `date::2026-04` mid-record, no + /// comma, no OHLCV fields, no newline. Atomic file writes correctly + /// persist that truncated body — `writeFileAtomic` is about rename + /// integrity, not payload validity. This helper is the payload- + /// validity check that cache-write callers should gate on when the + /// bytes come from an untrusted source (server sync, external file + /// import). Locally-serialized payloads are complete by construction + /// and don't need to run through this. + pub fn looksCompleteSrf(data: []const u8) bool { + if (data.len == 0) return false; + if (!std.mem.startsWith(u8, data, "#!srfv1")) return false; + if (data[data.len - 1] != '\n') return false; + return true; + } + /// Check if a cached data file is a negative entry (fetch_failed marker). /// Negative entries are always considered "fresh" -- they never expire. pub fn isNegative(self: *Store, symbol: []const u8, data_type: DataType) bool { @@ -1048,6 +1075,36 @@ test "negative_cache_content format" { try std.testing.expect(std.mem.indexOf(u8, Store.negative_cache_content, "fetch_failed") != null); } +test "looksCompleteSrf: empty is invalid" { + try std.testing.expect(!Store.looksCompleteSrf("")); +} + +test "looksCompleteSrf: missing srfv1 header rejected" { + try std.testing.expect(!Store.looksCompleteSrf("garbage\n")); + try std.testing.expect(!Store.looksCompleteSrf("some json body\n")); + try std.testing.expect(!Store.looksCompleteSrf("{\"error\":\"not found\"}\n")); +} + +test "looksCompleteSrf: missing trailing newline rejected" { + // Classic torn-write symptom: body ends mid-record with no newline. + // FRDM 2026-05-02 looked like this: trailing `date::2026-04` with + // no comma, no remaining fields, no `\n`. + try std.testing.expect(!Store.looksCompleteSrf( + "#!srfv1\ndate::2026-04-22,open:num:62.82\ndate::2026-04", + )); + try std.testing.expect(!Store.looksCompleteSrf("#!srfv1")); +} + +test "looksCompleteSrf: well-formed body accepted" { + try std.testing.expect(Store.looksCompleteSrf( + "#!srfv1\ndate::2026-04-22,open:num:62.82,close:num:63.23\n", + )); + // Negative cache content is also valid + try std.testing.expect(Store.looksCompleteSrf(Store.negative_cache_content)); + // Minimal: just the header with its own terminator + try std.testing.expect(Store.looksCompleteSrf("#!srfv1\n")); +} + test "Store.dataTypeFor maps model types correctly" { try std.testing.expectEqual(DataType.candles_daily, Store.dataTypeFor(Candle)); try std.testing.expectEqual(DataType.dividends, Store.dataTypeFor(Dividend)); diff --git a/src/service.zig b/src/service.zig index d3ef632..8bda2b9 100644 --- a/src/service.zig +++ b/src/service.zig @@ -1416,6 +1416,21 @@ pub const DataService = struct { }; defer response.deinit(); + // Validate the response body looks like a complete SRF file before + // writing it to cache. This guards against HTTP body truncation + // (TCP reset, Content-Length mismatch, proxy that flushed a + // partial response, etc.) — torn bodies get written atomically + // to the cache otherwise, producing the classic SRF parse error + // on the next read: + // error(srf): custom parse of value YYYY-MM failed : InvalidDateFormat + if (!cache.Store.looksCompleteSrf(response.body)) { + log.warn( + "{s}: rejecting torn {s} server response ({d} bytes) — not writing to cache", + .{ symbol, @tagName(data_type), response.body.len }, + ); + return false; + } + // Write to local cache var s = self.store(); s.writeRaw(symbol, data_type, response.body) catch |err| {