diff --git a/build.zig.zon b/build.zig.zon index a50f34c..5c9c1cb 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -14,8 +14,8 @@ .hash = "httpz-0.0.0-PNVzrLjJCAD37S0CcrXpsjSqr86hVjK0rsALTDJ98AAJ", }, .zfin = .{ - .url = "git+https://git.lerch.org/lobo/zfin#937822f9b04847942da4f57e9dfbde8cdd39be86", - .hash = "zfin-0.0.0-J-B21hCbVgAPhcCoDDQ5BAPkwN_Vk8is57gshMJO56F4", + .url = "git+https://git.lerch.org/lobo/zfin#4a86ecb95e1b155b58c2cdc294d850309bab934d", + .hash = "zfin-0.0.0-J-B21tBcVwCRYeqB7LbX67Ik9a2y8olgk7OObrUtJ8YU", }, }, } diff --git a/src/main.zig b/src/main.zig index 4294c22..88749dc 100644 --- a/src/main.zig +++ b/src/main.zig @@ -536,6 +536,142 @@ const SrfKind = enum { tickers_companies, }; +/// What the server INTENDS for a symbol, as opposed to the data it happens to +/// hold. Every other endpoint answers the second question; nothing answered the +/// first, which is how a symbol the refresh loop never touches sat six weeks +/// behind while being served to clients as though it were maintained. +const SymbolDiagnostics = struct { + /// Will the refresh loop fetch this symbol? See `collectRefreshSymbols`. + tracked: bool, + /// Newest cached bar, or null when there is no candle meta at all. + last_date: ?zfin.Date, + /// When the cached copy was written (Unix seconds), null when uncached. + created: ?i64, + /// Consecutive transient provider failures on the primary provider. + fail_count: u8, + /// Is the cached copy stamped fresh by its own `#!expires=`? Reported rather + /// than the raw expiry because `fresh` alongside a non-zero `days_behind` is + /// precisely the pathology that started this: a copy stamped good until + /// tomorrow while sitting days behind its peers. The raw directive is still + /// on the wire via `/:symbol/candles_meta` for anyone who wants it. + fresh: bool, + /// Newest bar held by any same-kind peer in this cache, or null when there is + /// no peer to compare against. + peer_date: ?zfin.Date, + /// Calendar days behind `peer_date`; 0 when not behind or incomparable. + days_behind: i64, +}; + +/// Calendar days `last` sits behind `peer`, or 0 when it is not behind. +/// +/// Computed here rather than read out of the sweep's findings. `scan` emits a +/// Finding only for a TRACKED symbol (untracked ones divert to `orphans`), so +/// reading `days_behind` from there returned 0 for every untracked symbol - +/// exactly the class this endpoint exists to expose. An untracked symbol sitting +/// 43 days behind, reported as `days_behind:0`, is the worst available answer: +/// it reads as "current" for the one case nobody is watching. +/// +/// Calendar days, truncated, to match `zfin.freshness.Finding.days_behind` - the +/// magnitude an operator weighs against `max_normal_lag_days`. +fn daysBehind(last: ?zfin.Date, peer: ?zfin.Date) i64 { + const l = last orelse return 0; + const p = peer orelse return 0; + if (!l.lessThan(p)) return 0; + return @divTrunc(p.toEpoch() - l.toEpoch(), std.time.s_per_day); +} + +/// The peer reference date for `kind`, or null when the group cannot yield a +/// comparison. `conclusive()` is the gate: with a single cached symbol of a kind +/// there are no peers, and reporting that symbol's own date as its `peer_date` +/// would manufacture agreement out of nothing. +fn groupPeerDate(report: zfin.freshness.Report, kind: zfin.market.InstrumentKind) ?zfin.Date { + for (report.groups) |g| { + if (g.kind != kind) continue; + if (!g.conclusive()) return null; + return g.peer_date; + } + return null; +} + +fn handleDiagnostics(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + const raw_symbol = req.param("symbol") orelse { + res.status = 400; + res.body = "Missing symbol"; + return; + }; + const arena = res.arena; + const symbol = try upperDupe(arena, raw_symbol); + + var store = zfin.cache.Store.init(app.io, arena, app.config.cache_dir); + + // An unreadable or unparseable portfolio leaves the set empty, which reports + // `tracked:false` - honest, because a refresh run reading the same file would + // fetch nothing either. + // + // NO `portfolio.deinit()` here, deliberately. The set's keys BORROW from the + // parsed lots, and every `contains` happens below this block, so freeing the + // portfolio dangles them - observed as all 25 tracked symbols reading as + // untracked. The request arena owns this memory and releases it with the + // response. This is the same trap `zfin`'s `trackedSymbols` documents: "the + // scoped-defer-frees version dangled its keys before the caller read them, + // which made every cached symbol look untracked." + var tracked = std.StringHashMap(void).init(arena); + const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; + if (std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024))) |data| { + if (zfin.cache.deserializePortfolio(arena, data)) |parsed| { + try collectRefreshSymbols(&tracked, parsed.lots); + } else |_| {} + } else |_| {} + + // The peer sweep. `collect` + `scan` rather than a local "newest of this + // kind" loop: the definition of behind-its-peers lives in one place, and this + // endpoint exists to report that definition, not a second opinion on it. + const keys = store.cacheKeys(arena) catch &.{}; + const entries = try zfin.freshness.collect(arena, &store, keys, &tracked, &.{}); + // wall-clock required: peer freshness is judged against the market calendar. + const now_s = std.Io.Timestamp.now(app.io, .real).toSeconds(); + const report = try zfin.freshness.scan(arena, entries, now_s); + + const meta = store.readCandleMeta(symbol); + const last_date: ?zfin.Date = if (meta) |m| m.meta.last_date else null; + const peer_date = groupPeerDate(report, zfin.market.classify(symbol)); + const d = SymbolDiagnostics{ + .tracked = tracked.contains(symbol), + .last_date = last_date, + .created = if (meta) |m| m.created else null, + .fail_count = if (meta) |m| m.meta.fail_count else 0, + .fresh = store.isCandleMetaFresh(symbol), + .peer_date = peer_date, + .days_behind = daysBehind(last_date, peer_date), + }; + + var aw: std.Io.Writer.Allocating = .init(arena); + try aw.writer.print( + \\{{"symbol":"{s}","tracked":{},"fresh":{},"fail_count":{d},"days_behind":{d} + , .{ symbol, d.tracked, d.fresh, d.fail_count, d.days_behind }); + // Null rather than a sentinel date for the absent cases: a client must be + // able to tell "no cached bar" from "a bar dated the epoch". + if (d.last_date) |ld| { + try aw.writer.print(",\"last_date\":\"{f}\"", .{ld}); + } else { + try aw.writer.writeAll(",\"last_date\":null"); + } + if (d.peer_date) |pd| { + try aw.writer.print(",\"peer_date\":\"{f}\"", .{pd}); + } else { + try aw.writer.writeAll(",\"peer_date\":null"); + } + if (d.created) |c| { + try aw.writer.print(",\"created\":{d}", .{c}); + } else { + try aw.writer.writeAll(",\"created\":null"); + } + try aw.writer.writeByte('}'); + + res.content_type = httpz.ContentType.JSON; + res.body = aw.written(); +} + fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void { return handleSrfFileByKey(app, req, res, "symbol", filename, kind); } @@ -951,6 +1087,22 @@ fn writeFileAtomic(io: std.Io, allocator: std.mem.Allocator, path: []const u8, b // ── Refresh command ────────────────────────────────────────── +/// The symbols the refresh loop will fetch: `.stock` and `.watch` lots, keyed by +/// `priceSymbol()` so a `ticker::` alias resolves the same way a fetch does. +/// +/// Extracted rather than inlined because `/:symbol/diagnostics` reports a +/// `tracked` flag, and a `tracked` that meant anything other than "refresh will +/// fetch this" would be worse than not reporting it at all - an operator would +/// read it as a promise the loop never made. One definition, two callers. +fn collectRefreshSymbols(out: *std.StringHashMap(void), lots: []const zfin.Lot) !void { + for (lots) |lot| { + if (lot.security_type != .stock and lot.security_type != .watch) continue; + if (lot.symbol.len == 0) continue; + const sym = lot.priceSymbol(); + if (!out.contains(sym)) try out.put(sym, {}); + } +} + fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) !u8 { var config = zfin.Config.fromEnv(io, allocator, environ); defer config.deinit(); @@ -983,14 +1135,7 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process var symbols = std.StringHashMap(void).init(allocator); defer symbols.deinit(); - for (portfolio.lots) |lot| { - if (lot.security_type != .stock and lot.security_type != .watch) continue; - if (lot.symbol.len == 0) continue; - const sym = lot.priceSymbol(); - if (!symbols.contains(sym)) { - try symbols.put(sym, {}); - } - } + try collectRefreshSymbols(&symbols, portfolio.lots); const stdout_file = std.Io.File.stdout(); var buf: [4096]u8 = undefined; @@ -1329,6 +1474,7 @@ pub fn main(init: std.process.Init) !u8 { router.get("/:symbol/quote", handleQuote, .{}); router.get("/:symbol/candles", handleCandles, .{}); router.get("/:symbol/candles_meta", handleCandlesMeta, .{}); + router.get("/:symbol/diagnostics", handleDiagnostics, .{}); router.get("/:symbol/dividends", handleDividends, .{}); router.get("/:symbol/splits", handleSplits, .{}); router.get("/:symbol/earnings", handleEarnings, .{}); @@ -1450,6 +1596,12 @@ test "pathIsPublic: gated surface (key required)" { try std.testing.expect(!pathIsPublic("/symbols")); try std.testing.expect(!pathIsPublic("/_edgar/tickers_funds")); try std.testing.expect(!pathIsPublic("/0000320193/entity_facts")); + // Diagnostics leaks the operator's tracked set and cache layout, so it must + // stay gated. It needs no entry in `pathIsPublic` to be gated - the default + // is closed - and this asserts the default rather than trusting it, because + // the cost of that assumption being wrong is silent disclosure. + try std.testing.expect(!pathIsPublic("/AAPL/diagnostics")); + try std.testing.expect(!pathIsPublic("/BRK.B/diagnostics")); } test "pathIsPublic: returns look-alikes do not slip through" { @@ -1533,3 +1685,122 @@ test "printSymbolList: empty shows (none), non-empty joins with commas" { defer std.testing.allocator.free(b_out); try std.testing.expectEqualStrings(" lagging: NKE, AMZN\n", b_out); } + +test "collectRefreshSymbols: only .stock and .watch, keyed by priceSymbol" { + const a = std.testing.allocator; + var set = std.StringHashMap(void).init(a); + defer set.deinit(); + + const lots = [_]zfin.Lot{ + .{ .symbol = "AAPL", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + .{ .symbol = "SPCX", .security_type = .watch, .shares = 0, .open_price = 0, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + // Cash is not fetched, so it must not appear as tracked - reporting it + // would promise a refresh that never runs. + .{ .symbol = "USD", .security_type = .cash, .shares = 100, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + // A duplicate holding is one symbol, not two. + .{ .symbol = "AAPL", .security_type = .stock, .shares = 5, .open_price = 5, .open_date = zfin.Date.fromYmd(2026, 2, 1) }, + // A CUSIP priced through a `ticker::` alias must register under the + // symbol a FETCH uses, not the one the statement shows - otherwise + // `tracked` says false for a symbol the loop refreshes every night. + .{ .symbol = "922908736", .ticker = "VTTHX", .security_type = .stock, .shares = 3, .open_price = 27, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + }; + try collectRefreshSymbols(&set, &lots); + + try std.testing.expectEqual(@as(u32, 3), set.count()); + try std.testing.expect(set.contains("AAPL")); + try std.testing.expect(set.contains("SPCX")); + try std.testing.expect(set.contains("VTTHX")); + try std.testing.expect(!set.contains("922908736")); + try std.testing.expect(!set.contains("USD")); +} + +test "daysBehind: an UNTRACKED symbol behind its peers still reports the gap" { + // The regression this replaces: `days_behind` was read out of + // `zfin.freshness.Report.stale`/`far_behind`, and `scan` only emits findings + // for tracked symbols - untracked ones divert to `orphans`. So every + // untracked symbol reported 0, and an untracked symbol is precisely what this + // endpoint was built to expose. AGG's real numbers, observed against a live + // cache copy: five days behind, reported as current. + try std.testing.expectEqual(@as(i64, 5), daysBehind( + zfin.Date.fromYmd(2026, 8, 6), + zfin.Date.fromYmd(2026, 8, 11), + )); + // SPCX's real gap. + try std.testing.expectEqual(@as(i64, 43), daysBehind( + zfin.Date.fromYmd(2026, 6, 29), + zfin.Date.fromYmd(2026, 8, 11), + )); +} + +test "daysBehind: not behind, or incomparable, is zero rather than negative" { + const d = zfin.Date.fromYmd(2026, 8, 11); + // Level with peers. + try std.testing.expectEqual(@as(i64, 0), daysBehind(d, d)); + // AHEAD of peers - this symbol IS the peer maximum. Must not report a + // negative gap, which would sort as "most behind" in any worst-first list. + try std.testing.expectEqual(@as(i64, 0), daysBehind(d, zfin.Date.fromYmd(2026, 8, 1))); + // No cached bar, and no peer to compare against: unanswerable, not zero-ish. + // Callers distinguish these from "current" via the null `last_date`. + try std.testing.expectEqual(@as(i64, 0), daysBehind(null, d)); + try std.testing.expectEqual(@as(i64, 0), daysBehind(d, null)); +} + +test "groupPeerDate: an inconclusive group has no peer date" { + const d = zfin.Date.fromYmd(2026, 8, 11); + // `dated = 1` is the symbol itself and nothing else. Returning its own date + // as `peer_date` would read as "agrees with its peers" when there are none. + const lonely = [_]zfin.freshness.GroupState{.{ + .kind = .equity, + .peer_date = d, + .freshness = null, + .dated = 1, + }}; + var report = zfin.freshness.Report{ + .stale = &.{}, + .far_behind = &.{}, + .orphans = &.{}, + .missing = &.{}, + .groups = @constCast(lonely[0..]), + }; + try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .equity)); + + // Two dated entries make a comparison possible. + const peers = [_]zfin.freshness.GroupState{.{ + .kind = .equity, + .peer_date = d, + .freshness = null, + .dated = 2, + }}; + report.groups = @constCast(peers[0..]); + try std.testing.expectEqual(@as(?zfin.Date, d), groupPeerDate(report, .equity)); + + // A kind with no group at all is not an error, just unanswerable. + try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .mutual_fund)); +} + +test "collectRefreshSymbols: keys stay readable while the source lots live" { + // Guards the bug this shipped with: the handler parsed the portfolio in an + // inner scope with a `defer portfolio.deinit()`, so by the time it asked + // `tracked.contains(symbol)` the keys pointed at released memory and all 25 + // tracked symbols reported false. `zfin`'s own `trackedSymbols` carries a + // comment about the identical failure - "made every cached symbol look + // untracked" - which is what makes it worth a test rather than a comment. + // + // The invariant is ownership, not content: the set BORROWS from `lots`, so a + // lookup is only valid while `lots` is alive. Asserting a hit after the + // insert-scope has closed is the cheapest way to pin that. + const a = std.testing.allocator; + var set = std.StringHashMap(void).init(a); + defer set.deinit(); + + const lots = [_]zfin.Lot{ + .{ .symbol = "AMZN", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + }; + { + // A nested scope that ends before the lookup, mirroring the handler's + // shape. `lots` outlives it, so the keys remain valid. + try collectRefreshSymbols(&set, &lots); + } + try std.testing.expect(set.contains("AMZN")); + try std.testing.expectEqual(@as(u32, 1), set.count()); +}