address symptom of Torn SRF files while we track this down
This commit is contained in:
parent
8b6e10ea10
commit
2ac1d30c37
3 changed files with 90 additions and 0 deletions
18
TODO.md
18
TODO.md
|
|
@ -197,3 +197,21 @@ Only matters with very large portfolios (100+ symbols) hitting ZFIN_SERVER.
|
||||||
Cache store reads entire files into memory (`readFileAlloc` with 50MB limit).
|
Cache store reads entire files into memory (`readFileAlloc` with 50MB limit).
|
||||||
For portfolios with 10+ years of daily candles, this could use significant
|
For portfolios with 10+ years of daily candles, this could use significant
|
||||||
memory. Keep current approach unless memory becomes a real problem.
|
memory. Keep current approach unless memory becomes a real problem.
|
||||||
|
|
||||||
|
## Torn SRF files from server sync (recurring bug)
|
||||||
|
|
||||||
|
**Status:** First-layer fix done — 2026-05-02. `syncFromServer`
|
||||||
|
(`src/service.zig`) now validates responses via
|
||||||
|
`cache.Store.looksCompleteSrf` before `writeRaw`. Torn HTTP bodies
|
||||||
|
(empty, missing `#!srfv1` header, or no trailing newline) are
|
||||||
|
rejected with a warn-level log and NOT written to cache. Next fetch
|
||||||
|
will try the provider fallback.
|
||||||
|
|
||||||
|
**Remaining work (if it comes back):**
|
||||||
|
|
||||||
|
- HTTP-level Content-Length validation in `src/net/http.zig` to fail
|
||||||
|
closer to the source rather than at the cache write.
|
||||||
|
- Read-path self-heal: on SRF parse failure during read, invalidate
|
||||||
|
the cache entry so a subsequent refresh can repair without user
|
||||||
|
intervention.
|
||||||
|
|
||||||
|
|
|
||||||
57
src/cache/store.zig
vendored
57
src/cache/store.zig
vendored
|
|
@ -316,6 +316,33 @@ pub const Store = struct {
|
||||||
self.writeRaw(symbol, data_type, negative_cache_content) catch {};
|
self.writeRaw(symbol, data_type, negative_cache_content) catch {};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Validate that a byte buffer looks like a complete SRF file.
|
||||||
|
///
|
||||||
|
/// A well-formed SRF file should:
|
||||||
|
/// - Be non-empty.
|
||||||
|
/// - Start with the `#!srfv1` version directive (the negative-cache
|
||||||
|
/// marker also starts this way, so a single check covers both).
|
||||||
|
/// - End with a newline. Every record and every directive is
|
||||||
|
/// newline-terminated; a completely-written file always has a
|
||||||
|
/// trailing `\n`.
|
||||||
|
///
|
||||||
|
/// Torn HTTP body writes land short of the final newline. The FRDM
|
||||||
|
/// corruption on 2026-05-02 is the canonical example: the tail of
|
||||||
|
/// `candles_daily.srf` ended with `date::2026-04` mid-record, no
|
||||||
|
/// comma, no OHLCV fields, no newline. Atomic file writes correctly
|
||||||
|
/// persist that truncated body — `writeFileAtomic` is about rename
|
||||||
|
/// integrity, not payload validity. This helper is the payload-
|
||||||
|
/// validity check that cache-write callers should gate on when the
|
||||||
|
/// bytes come from an untrusted source (server sync, external file
|
||||||
|
/// import). Locally-serialized payloads are complete by construction
|
||||||
|
/// and don't need to run through this.
|
||||||
|
pub fn looksCompleteSrf(data: []const u8) bool {
|
||||||
|
if (data.len == 0) return false;
|
||||||
|
if (!std.mem.startsWith(u8, data, "#!srfv1")) return false;
|
||||||
|
if (data[data.len - 1] != '\n') return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/// Check if a cached data file is a negative entry (fetch_failed marker).
|
/// Check if a cached data file is a negative entry (fetch_failed marker).
|
||||||
/// Negative entries are always considered "fresh" -- they never expire.
|
/// Negative entries are always considered "fresh" -- they never expire.
|
||||||
pub fn isNegative(self: *Store, symbol: []const u8, data_type: DataType) bool {
|
pub fn isNegative(self: *Store, symbol: []const u8, data_type: DataType) bool {
|
||||||
|
|
@ -1048,6 +1075,36 @@ test "negative_cache_content format" {
|
||||||
try std.testing.expect(std.mem.indexOf(u8, Store.negative_cache_content, "fetch_failed") != null);
|
try std.testing.expect(std.mem.indexOf(u8, Store.negative_cache_content, "fetch_failed") != null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "looksCompleteSrf: empty is invalid" {
|
||||||
|
try std.testing.expect(!Store.looksCompleteSrf(""));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "looksCompleteSrf: missing srfv1 header rejected" {
|
||||||
|
try std.testing.expect(!Store.looksCompleteSrf("garbage\n"));
|
||||||
|
try std.testing.expect(!Store.looksCompleteSrf("some json body\n"));
|
||||||
|
try std.testing.expect(!Store.looksCompleteSrf("{\"error\":\"not found\"}\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "looksCompleteSrf: missing trailing newline rejected" {
|
||||||
|
// Classic torn-write symptom: body ends mid-record with no newline.
|
||||||
|
// FRDM 2026-05-02 looked like this: trailing `date::2026-04` with
|
||||||
|
// no comma, no remaining fields, no `\n`.
|
||||||
|
try std.testing.expect(!Store.looksCompleteSrf(
|
||||||
|
"#!srfv1\ndate::2026-04-22,open:num:62.82\ndate::2026-04",
|
||||||
|
));
|
||||||
|
try std.testing.expect(!Store.looksCompleteSrf("#!srfv1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "looksCompleteSrf: well-formed body accepted" {
|
||||||
|
try std.testing.expect(Store.looksCompleteSrf(
|
||||||
|
"#!srfv1\ndate::2026-04-22,open:num:62.82,close:num:63.23\n",
|
||||||
|
));
|
||||||
|
// Negative cache content is also valid
|
||||||
|
try std.testing.expect(Store.looksCompleteSrf(Store.negative_cache_content));
|
||||||
|
// Minimal: just the header with its own terminator
|
||||||
|
try std.testing.expect(Store.looksCompleteSrf("#!srfv1\n"));
|
||||||
|
}
|
||||||
|
|
||||||
test "Store.dataTypeFor maps model types correctly" {
|
test "Store.dataTypeFor maps model types correctly" {
|
||||||
try std.testing.expectEqual(DataType.candles_daily, Store.dataTypeFor(Candle));
|
try std.testing.expectEqual(DataType.candles_daily, Store.dataTypeFor(Candle));
|
||||||
try std.testing.expectEqual(DataType.dividends, Store.dataTypeFor(Dividend));
|
try std.testing.expectEqual(DataType.dividends, Store.dataTypeFor(Dividend));
|
||||||
|
|
|
||||||
|
|
@ -1416,6 +1416,21 @@ pub const DataService = struct {
|
||||||
};
|
};
|
||||||
defer response.deinit();
|
defer response.deinit();
|
||||||
|
|
||||||
|
// Validate the response body looks like a complete SRF file before
|
||||||
|
// writing it to cache. This guards against HTTP body truncation
|
||||||
|
// (TCP reset, Content-Length mismatch, proxy that flushed a
|
||||||
|
// partial response, etc.) — torn bodies get written atomically
|
||||||
|
// to the cache otherwise, producing the classic SRF parse error
|
||||||
|
// on the next read:
|
||||||
|
// error(srf): custom parse of value YYYY-MM failed : InvalidDateFormat
|
||||||
|
if (!cache.Store.looksCompleteSrf(response.body)) {
|
||||||
|
log.warn(
|
||||||
|
"{s}: rejecting torn {s} server response ({d} bytes) — not writing to cache",
|
||||||
|
.{ symbol, @tagName(data_type), response.body.len },
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
// Write to local cache
|
// Write to local cache
|
||||||
var s = self.store();
|
var s = self.store();
|
||||||
s.writeRaw(symbol, data_type, response.body) catch |err| {
|
s.writeRaw(symbol, data_type, response.body) catch |err| {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue