address symptom of Torn SRF files while we track this down

This commit is contained in:
Emil Lerch 2026-05-02 10:07:59 -07:00
parent 94311f6ff7
commit 7a05d53dc9
Signed by: lobo
GPG key ID: A7B62D657EF764F8
3 changed files with 90 additions and 0 deletions

18
TODO.md
View file

@ -197,3 +197,21 @@ Only matters with very large portfolios (100+ symbols) hitting ZFIN_SERVER.
Cache store reads entire files into memory (`readFileAlloc` with 50MB limit).
For portfolios with 10+ years of daily candles, this could use significant
memory. Keep current approach unless memory becomes a real problem.
## Torn SRF files from server sync (recurring bug)
**Status:** First-layer fix done — 2026-05-02. `syncFromServer`
(`src/service.zig`) now validates responses via
`cache.Store.looksCompleteSrf` before `writeRaw`. Torn HTTP bodies
(empty, missing `#!srfv1` header, or no trailing newline) are
rejected with a warn-level log and NOT written to cache. Next fetch
will try the provider fallback.
**Remaining work (if it comes back):**
- HTTP-level Content-Length validation in `src/net/http.zig` to fail
closer to the source rather than at the cache write.
- Read-path self-heal: on SRF parse failure during read, invalidate
the cache entry so a subsequent refresh can repair without user
intervention.

57
src/cache/store.zig vendored
View file

@ -316,6 +316,33 @@ pub const Store = struct {
self.writeRaw(symbol, data_type, negative_cache_content) catch {};
}
/// Validate that a byte buffer looks like a complete SRF file.
///
/// A well-formed SRF file should:
/// - Be non-empty.
/// - Start with the `#!srfv1` version directive (the negative-cache
/// marker also starts this way, so a single check covers both).
/// - End with a newline. Every record and every directive is
/// newline-terminated; a completely-written file always has a
/// trailing `\n`.
///
/// Torn HTTP body writes land short of the final newline. The FRDM
/// corruption on 2026-05-02 is the canonical example: the tail of
/// `candles_daily.srf` ended with `date::2026-04` mid-record, no
/// comma, no OHLCV fields, no newline. Atomic file writes correctly
/// persist that truncated body `writeFileAtomic` is about rename
/// integrity, not payload validity. This helper is the payload-
/// validity check that cache-write callers should gate on when the
/// bytes come from an untrusted source (server sync, external file
/// import). Locally-serialized payloads are complete by construction
/// and don't need to run through this.
pub fn looksCompleteSrf(data: []const u8) bool {
if (data.len == 0) return false;
if (!std.mem.startsWith(u8, data, "#!srfv1")) return false;
if (data[data.len - 1] != '\n') return false;
return true;
}
/// Check if a cached data file is a negative entry (fetch_failed marker).
/// Negative entries are always considered "fresh" -- they never expire.
pub fn isNegative(self: *Store, symbol: []const u8, data_type: DataType) bool {
@ -1048,6 +1075,36 @@ test "negative_cache_content format" {
try std.testing.expect(std.mem.indexOf(u8, Store.negative_cache_content, "fetch_failed") != null);
}
test "looksCompleteSrf: empty is invalid" {
try std.testing.expect(!Store.looksCompleteSrf(""));
}
test "looksCompleteSrf: missing srfv1 header rejected" {
try std.testing.expect(!Store.looksCompleteSrf("garbage\n"));
try std.testing.expect(!Store.looksCompleteSrf("some json body\n"));
try std.testing.expect(!Store.looksCompleteSrf("{\"error\":\"not found\"}\n"));
}
test "looksCompleteSrf: missing trailing newline rejected" {
// Classic torn-write symptom: body ends mid-record with no newline.
// FRDM 2026-05-02 looked like this: trailing `date::2026-04` with
// no comma, no remaining fields, no `\n`.
try std.testing.expect(!Store.looksCompleteSrf(
"#!srfv1\ndate::2026-04-22,open:num:62.82\ndate::2026-04",
));
try std.testing.expect(!Store.looksCompleteSrf("#!srfv1"));
}
test "looksCompleteSrf: well-formed body accepted" {
try std.testing.expect(Store.looksCompleteSrf(
"#!srfv1\ndate::2026-04-22,open:num:62.82,close:num:63.23\n",
));
// Negative cache content is also valid
try std.testing.expect(Store.looksCompleteSrf(Store.negative_cache_content));
// Minimal: just the header with its own terminator
try std.testing.expect(Store.looksCompleteSrf("#!srfv1\n"));
}
test "Store.dataTypeFor maps model types correctly" {
try std.testing.expectEqual(DataType.candles_daily, Store.dataTypeFor(Candle));
try std.testing.expectEqual(DataType.dividends, Store.dataTypeFor(Dividend));

View file

@ -1416,6 +1416,21 @@ pub const DataService = struct {
};
defer response.deinit();
// Validate the response body looks like a complete SRF file before
// writing it to cache. This guards against HTTP body truncation
// (TCP reset, Content-Length mismatch, proxy that flushed a
// partial response, etc.) torn bodies get written atomically
// to the cache otherwise, producing the classic SRF parse error
// on the next read:
// error(srf): custom parse of value YYYY-MM failed : InvalidDateFormat
if (!cache.Store.looksCompleteSrf(response.body)) {
log.warn(
"{s}: rejecting torn {s} server response ({d} bytes) — not writing to cache",
.{ symbol, @tagName(data_type), response.body.len },
);
return false;
}
// Write to local cache
var s = self.store();
s.writeRaw(symbol, data_type, response.body) catch |err| {