From 79326cf41bbfee0a2129cd0c9ad77c37a40c5481 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Tue, 11 Aug 2026 09:49:39 -0700 Subject: [PATCH] add a zfin cache stale command to find out of date candle data --- src/cache/freshness.zig | 434 ++++++++++++++++++++++++++++++++++++++++ src/commands/cache.zig | 212 +++++++++++++++++++- src/commands/common.zig | 24 +++ src/commands/doctor.zig | 28 +-- 4 files changed, 673 insertions(+), 25 deletions(-) create mode 100644 src/cache/freshness.zig diff --git a/src/cache/freshness.zig b/src/cache/freshness.zig new file mode 100644 index 0000000..395e3ab --- /dev/null +++ b/src/cache/freshness.zig @@ -0,0 +1,434 @@ +//! Corpus-wide candle-cache staleness detection. +//! +//! `market.candleFreshness` answers "is THIS symbol's newest bar overdue?" from +//! the clock alone. That is the right question for a fetch gate and the wrong +//! one for finding a symbol the cache has quietly frozen, because the clock +//! cannot distinguish three cases that look identical per-symbol: the bar isn't +//! published yet, the market was closed, or this symbol alone got left behind. +//! +//! Peers settle it. Symbols of the same `InstrumentKind` share a publication +//! schedule, so if twenty equities hold Friday's bar and three hold Thursday's, +//! those three are behind - no calendar reasoning required, and an un-modeled +//! closure cannot fool it because a closure moves every symbol together, taking +//! the peer maximum with them and leaving nothing behind it. +//! +//! That self-correction is why there is no calendar clamp on top. A first cut +//! suppressed findings whenever the group's own newest bar was itself overdue, +//! reasoning that a wholly-behind group meant a closure. It meant nothing of +//! the kind: it meant the corpus had not been refreshed yet that day, which is +//! the normal state, and the gate hid three genuinely-lagging symbols in live +//! data. "Is the corpus caught up?" and "is any symbol behind the rest?" are +//! independent questions, and only the second one names a symbol. +//! +//! The observed failure this exists for: three symbols served Thursday's close +//! all weekend while ~19 peers had Friday's. Each had been fetched, missed, +//! stamped `.lagging`, retried 30 minutes later, and by then sat just past the +//! 90-minute provider-lag grace - so `candleFreshness` said `.overdue`, +//! `staleCandleExpiry` routed them to the next session boundary, and the stale +//! bars became authoritative until Monday. The retry machinery worked as +//! designed; the premise was wrong. +//! +//! Pure: dates and `now_s` in, findings out. No I/O, no cache reads, no +//! fetches. The caller supplies the corpus. + +const std = @import("std"); +const Date = @import("../Date.zig"); +const market = @import("../market.zig"); + +/// One cached symbol, as the caller found it. +pub const Entry = struct { + symbol: []const u8, + kind: market.InstrumentKind, + /// Newest bar in the cache, or null when there is no candle meta at all - + /// never fetched, negative-cached, or a key that carries no candles (an + /// EDGAR CIK, say). Null entries are never reported stale: absence of + /// evidence is not a stale bar. + last_date: ?Date, + /// Is this symbol in the set zfin intends to keep fresh + /// (`Portfolio.fetchedSymbols`)? An untracked symbol can only ever fall + /// further behind, so reporting it as stale every run would be noise. + tracked: bool, +}; + +/// A tracked symbol whose newest bar is behind its peers'. +pub const Finding = struct { + symbol: []const u8, + kind: market.InstrumentKind, + last_date: Date, + /// Newest bar held by any same-kind peer. + peer_date: Date, + /// Calendar days between the two. Calendar rather than trading days + /// because this is a human-facing magnitude, and the distinction between + /// "one session" and "six weeks" is what it has to convey. + days_behind: i64, + + fn worstFirst(_: void, a: Finding, b: Finding) bool { + if (a.days_behind != b.days_behind) return a.days_behind > b.days_behind; + return std.mem.order(u8, a.symbol, b.symbol) == .lt; + } +}; + +/// Per-`InstrumentKind` reference state, so a caller can explain itself rather +/// than just listing symbols. +pub const GroupState = struct { + kind: market.InstrumentKind, + /// Newest bar across every dated entry of this kind; null when the kind + /// has no dated entries. + peer_date: ?Date, + /// Freshness of the group's OWN newest bar - context, NOT a gate. + /// + /// Reported so a caller can say "the whole corpus is a session behind, + /// refresh everything" alongside any per-symbol findings. It deliberately + /// does not suppress those findings: an earlier version gated on + /// `.current` and thereby hid three genuinely-lagging symbols every time + /// the corpus as a whole had not been refreshed that day, which is most of + /// the time. + /// + /// No clamp is needed, because peer comparison is already immune to the + /// case a clamp was meant to catch: a market closure moves every symbol + /// together, so the peer maximum moves with them and nothing sits behind + /// it. The two questions are independent - "is the corpus caught up?" and + /// "is any symbol behind the rest?" - and only the second one identifies a + /// symbol. + freshness: ?market.CandleFreshness, + /// Dated entries considered for this kind. + dated: usize, + + /// Can this kind yield findings at all? Only that a peer exists to compare + /// against; with a single cached symbol there is no comparison to make. + pub fn conclusive(self: GroupState) bool { + return self.dated > 1; + } +}; + +pub const Report = struct { + /// Tracked symbols behind their peers, worst first. + stale: []Finding, + /// Cached, dated, but tracked by nothing. Nothing will ever refresh these, + /// so they are not staleness - they are disk you may want back. Reported + /// separately precisely so they cannot dominate the stale list forever. + orphans: []const []const u8, + /// Tracked but with no cached bar at all. A different finding: not stale, + /// never fetched. + missing: []const []const u8, + groups: []GroupState, + + pub fn deinit(self: Report, allocator: std.mem.Allocator) void { + allocator.free(self.stale); + allocator.free(self.orphans); + allocator.free(self.missing); + allocator.free(self.groups); + } +}; + +/// Classify `entries` as of `now_s`. +/// +/// Symbol strings are borrowed from `entries`; only the slices are allocated. +pub fn scan( + allocator: std.mem.Allocator, + entries: []const Entry, + now_s: i64, +) !Report { + const kinds = [_]market.InstrumentKind{ .equity, .mutual_fund }; + + var groups = try allocator.alloc(GroupState, kinds.len); + errdefer allocator.free(groups); + for (kinds, 0..) |kind, i| { + var newest: ?Date = null; + var dated: usize = 0; + for (entries) |e| { + if (e.kind != kind) continue; + const d = e.last_date orelse continue; + dated += 1; + if (newest == null or newest.?.lessThan(d)) newest = d; + } + groups[i] = .{ + .kind = kind, + .peer_date = newest, + // Context only (see GroupState.freshness). Reuses the fetch + // gate's own verdict rather than re-deriving the market calendar. + .freshness = if (newest) |n| market.candleFreshness(now_s, kind, n) else null, + .dated = dated, + }; + } + + var stale = std.ArrayList(Finding).empty; + errdefer stale.deinit(allocator); + var orphans = std.ArrayList([]const u8).empty; + errdefer orphans.deinit(allocator); + var missing = std.ArrayList([]const u8).empty; + errdefer missing.deinit(allocator); + + for (entries) |e| { + const last = e.last_date orelse { + // No bar at all. Only worth saying when something tracks it. + if (e.tracked) try missing.append(allocator, e.symbol); + continue; + }; + if (!e.tracked) { + try orphans.append(allocator, e.symbol); + continue; + } + + const g = groups[if (e.kind == .equity) 0 else 1]; + if (!g.conclusive()) continue; + const peer = g.peer_date.?; + if (!last.lessThan(peer)) continue; + + try stale.append(allocator, .{ + .symbol = e.symbol, + .kind = e.kind, + .last_date = last, + .peer_date = peer, + .days_behind = @divTrunc(peer.toEpoch() - last.toEpoch(), std.time.s_per_day), + }); + } + + // Ownership moves out of the ArrayLists one at a time, so each completed + // handoff needs its own errdefer: the list's errdefer no longer covers a + // slice it has released, and the Report does not exist yet to cover it + // either. An allocation-failure test caught `stale_out` leaking in exactly + // that window. + const stale_out = try stale.toOwnedSlice(allocator); + errdefer allocator.free(stale_out); + std.mem.sort(Finding, stale_out, {}, Finding.worstFirst); + + const orphans_out = try orphans.toOwnedSlice(allocator); + errdefer allocator.free(orphans_out); + + const missing_out = try missing.toOwnedSlice(allocator); + + return .{ + .stale = stale_out, + .orphans = orphans_out, + .missing = missing_out, + .groups = groups, + }; +} + +// ── tests ──────────────────────────────────────────────────── + +const testing = std.testing; + +/// Friday 2025-06-13 at 18:00 ET, past the 16:55 equity boundary. Equities +/// should hold Friday's bar; mutual-fund NAVs for Friday are not published +/// until the next morning, so funds should hold Thursday's. +fn fridayEvening() i64 { + return Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour; +} + +fn eq(sym: []const u8, d: ?Date, tracked: bool) Entry { + return .{ .symbol = sym, .kind = .equity, .last_date = d, .tracked = tracked }; +} + +test "scan: a symbol behind its peers is reported" { + const a = testing.allocator; + // The shape of the observed failure: most of the corpus has the latest + // bar, a few do not. + const fri = Date.fromYmd(2025, 6, 13); + const thu = Date.fromYmd(2025, 6, 12); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("NVDA", fri, true), + eq("TSLA", thu, true), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 1), r.stale.len); + try testing.expectEqualStrings("TSLA", r.stale[0].symbol); + try testing.expectEqual(@as(i64, 1), r.stale[0].days_behind); + try testing.expect(r.stale[0].peer_date.eql(fri)); +} + +test "scan: a uniformly-behind group reports nothing, and needs no clamp to do it" { + const a = testing.allocator; + // A closure (or simply a corpus nobody refreshed) moves every symbol + // together, so the peer maximum moves too and no symbol sits behind it. + // Peer comparison is self-correcting here - a clock-based check would flag + // all four, and an added calendar clamp would suppress real findings + // elsewhere for no gain. `freshness` is still reported as context so a + // caller can say "refresh everything". + const wed = Date.fromYmd(2025, 6, 11); + const entries = [_]Entry{ + eq("AAPL", wed, true), + eq("MSFT", wed, true), + eq("NVDA", wed, true), + eq("TSLA", wed, true), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 0), r.stale.len); + // Context says the corpus is behind; it does not gate anything. + try testing.expect(r.groups[0].freshness.? != .current); + try testing.expect(r.groups[0].conclusive()); +} + +test "scan: a laggard is found even when the corpus as a whole is behind" { + const a = testing.allocator; + // THE REGRESSION THIS GUARDS, caught on live data. Monday evening, nothing + // has refreshed since Friday: every symbol is behind the session the + // calendar expects. An earlier version gated findings on the group's own + // bar being `.current` and so reported nothing at all - while three + // symbols sat a full session behind the other twenty-one. + // + // Monday 2025-06-16 22:00 UTC, past the equity boundary, so Monday's bar + // is expected and even the peer maximum (Friday) is overdue. + const monday_evening = Date.fromYmd(2025, 6, 16).toEpoch() + 22 * std.time.s_per_hour; + const fri = Date.fromYmd(2025, 6, 13); + const thu = Date.fromYmd(2025, 6, 12); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("NVDA", fri, true), + eq("AGG", thu, true), + eq("AMZN", thu, true), + eq("NKE", thu, true), + }; + var r = try scan(a, &entries, monday_evening); + defer r.deinit(a); + + // The group is behind - and that must not silence the three outliers. + try testing.expect(r.groups[0].freshness.? != .current); + try testing.expectEqual(@as(usize, 3), r.stale.len); + for (r.stale) |f| { + try testing.expect(f.last_date.eql(thu)); + try testing.expect(f.peer_date.eql(fri)); + try testing.expectEqual(@as(i64, 1), f.days_behind); + } +} + +test "scan: kinds are judged separately, so fund NAV lag is not staleness" { + const a = testing.allocator; + // Friday evening: equities have Friday, funds still legitimately have + // Thursday. Judged as one corpus, every fund would read a day stale every + // single evening. + const fri = Date.fromYmd(2025, 6, 13); + const thu = Date.fromYmd(2025, 6, 12); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + .{ .symbol = "VBTLX", .kind = .mutual_fund, .last_date = thu, .tracked = true }, + .{ .symbol = "VPMAX", .kind = .mutual_fund, .last_date = thu, .tracked = true }, + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 0), r.stale.len); + // Both groups are caught up against their own schedules. + try testing.expectEqual(market.CandleFreshness.current, r.groups[0].freshness.?); + try testing.expectEqual(market.CandleFreshness.current, r.groups[1].freshness.?); +} + +test "scan: a lone symbol in its kind yields no verdict" { + const a = testing.allocator; + // With one peer there is nothing to compare against, and the clock alone + // cannot tell a frozen cache from a closed market. Silence is the honest + // answer, and this is peer comparison's documented blind spot. + const entries = [_]Entry{ + .{ .symbol = "VBTLX", .kind = .mutual_fund, .last_date = Date.fromYmd(2025, 5, 1), .tracked = true }, + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + try testing.expectEqual(@as(usize, 0), r.stale.len); + try testing.expect(!r.groups[1].conclusive()); + try testing.expectEqual(@as(usize, 1), r.groups[1].dated); +} + +test "scan: an untracked symbol is an orphan, never stale" { + const a = testing.allocator; + // Nothing refreshes it, so it can only fall further behind. Reporting it + // as stale would put a permanent entry at the top of the list and train + // the operator to skip the section. + const fri = Date.fromYmd(2025, 6, 13); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("OLDCO", Date.fromYmd(2025, 1, 2), false), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 0), r.stale.len); + try testing.expectEqual(@as(usize, 1), r.orphans.len); + try testing.expectEqualStrings("OLDCO", r.orphans[0]); +} + +test "scan: no candle meta means missing when tracked, ignored when not" { + const a = testing.allocator; + // A negative-cached symbol or a non-candle key has no bar. That is not a + // stale bar, and inventing an infinitely-old date for it would put it + // permanently at the top of the worst-first list. + const fri = Date.fromYmd(2025, 6, 13); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("DOGE-USD", null, true), + eq("0000320193", null, false), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 0), r.stale.len); + try testing.expectEqual(@as(usize, 0), r.orphans.len); + try testing.expectEqual(@as(usize, 1), r.missing.len); + try testing.expectEqualStrings("DOGE-USD", r.missing[0]); + try testing.expectEqual(@as(usize, 2), r.groups[0].dated); +} + +test "scan: findings are ordered worst first" { + const a = testing.allocator; + // A symbol one session behind and one six weeks behind are different + // diagnoses; the second should not be buried under the first. + const fri = Date.fromYmd(2025, 6, 13); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("NEAR", Date.fromYmd(2025, 6, 12), true), + eq("FAR", Date.fromYmd(2025, 5, 2), true), + eq("MID", Date.fromYmd(2025, 6, 6), true), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 3), r.stale.len); + try testing.expectEqualStrings("FAR", r.stale[0].symbol); + try testing.expectEqualStrings("MID", r.stale[1].symbol); + try testing.expectEqualStrings("NEAR", r.stale[2].symbol); + try testing.expectEqual(@as(i64, 42), r.stale[0].days_behind); +} + +test "scan: an empty corpus is not an error" { + const a = testing.allocator; + var r = try scan(a, &.{}, fridayEvening()); + defer r.deinit(a); + try testing.expectEqual(@as(usize, 0), r.stale.len); + for (r.groups) |g| { + try testing.expectEqual(@as(?Date, null), g.peer_date); + try testing.expect(!g.conclusive()); + } +} + +/// OOM-path wrapper for `checkAllAllocationFailures`. +fn scanOom(a: std.mem.Allocator, entries: []const Entry, now_s: i64) !void { + var r = try scan(a, entries, now_s); + r.deinit(a); +} + +test "scan: every allocation-failure path unwinds cleanly" { + const fri = Date.fromYmd(2025, 6, 13); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("TSLA", Date.fromYmd(2025, 6, 12), true), + eq("OLDCO", Date.fromYmd(2025, 1, 2), false), + eq("DOGE-USD", null, true), + }; + try testing.checkAllAllocationFailures( + testing.allocator, + scanOom, + .{ @as([]const Entry, &entries), fridayEvening() }, + ); +} diff --git a/src/commands/cache.zig b/src/commands/cache.zig index a340bbd..e43e84d 100644 --- a/src/commands/cache.zig +++ b/src/commands/cache.zig @@ -3,11 +3,13 @@ const zfin = @import("../root.zig"); const cli = @import("common.zig"); const framework = @import("framework.zig"); const srf = @import("srf"); +const freshness = @import("../cache/freshness.zig"); +const projections = @import("../analytics/projections.zig"); const Store = zfin.cache.Store; const DataType = zfin.cache.DataType; -pub const Subcommand = enum { stats, clear }; +pub const Subcommand = enum { stats, stale, clear }; pub const ParsedArgs = struct { sub: Subcommand, @@ -18,13 +20,19 @@ pub const meta: framework.Meta = .{ .group = .infra, .synopsis = "Inspect or clear the local provider-data cache", .help = - \\Usage: zfin cache + \\Usage: zfin cache \\ \\Subcommands: \\ stats List every cached symbol with per-data-type size, \\ age, and freshness state. Stale entries (past TTL) \\ are flagged. Includes the cusip_tickers.srf file \\ if present. + \\ stale Find symbols whose newest candle is behind their + \\ peers'. Compares each symbol against others of the + \\ same kind (equity vs mutual fund) rather than against + \\ the clock, so an un-modeled market closure - which + \\ moves every symbol together - is not mistaken for a + \\ frozen cache. Read-only; reports, never fetches. \\ clear Delete every file under the cache directory. \\ No confirmation; the next provider call will \\ re-fetch everything. @@ -55,7 +63,7 @@ const display_labels = [_][]const u8{ pub fn parseArgs(ctx: *framework.RunCtx, cmd_args: []const []const u8) !ParsedArgs { if (cmd_args.len < 1) { - cli.stderrPrint(ctx.io, "Error: 'cache' requires a subcommand (stats, clear)\n"); + cli.stderrPrint(ctx.io, "Error: 'cache' requires a subcommand (stats, stale, clear)\n"); return error.MissingSubcommand; } if (cmd_args.len > 1) { @@ -66,18 +74,22 @@ pub fn parseArgs(ctx: *framework.RunCtx, cmd_args: []const []const u8) !ParsedAr if (std.mem.eql(u8, sub_str, "stats")) { return .{ .sub = .stats }; } + if (std.mem.eql(u8, sub_str, "stale")) { + return .{ .sub = .stale }; + } if (std.mem.eql(u8, sub_str, "clear")) { return .{ .sub = .clear }; } cli.stderrPrint(ctx.io, "Error: unknown cache subcommand '"); cli.stderrPrint(ctx.io, sub_str); - cli.stderrPrint(ctx.io, "'. Use 'stats' or 'clear'.\n"); + cli.stderrPrint(ctx.io, "'. Use 'stats', 'stale' or 'clear'.\n"); return error.UnknownSubcommand; } pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void { switch (parsed.sub) { .stats => try runStats(ctx), + .stale => try runStale(ctx), .clear => try runClear(ctx), } } @@ -191,6 +203,147 @@ fn runStats(ctx: *framework.RunCtx) !void { }); } +/// Peer-comparison staleness sweep over the cached corpus. +/// +/// Read-only by design: it reports and never fetches, so it is safe to run +/// against a shared cache and safe to put in front of an operator who has not +/// decided what to do yet. `zfin cache refresh` is the acting half. +fn runStale(ctx: *framework.RunCtx) !void { + const io = ctx.io; + const allocator = ctx.allocator; + const out = ctx.out; + + // wall-clock required: the sweep compares each symbol's newest bar against + // the market calendar's notion of what should be published by now. + // Captured once so every symbol is judged against the same instant. + const now_s = std.Io.Timestamp.now(io, .real).toSeconds(); + + // Arena for the transient string work below. Not a nicety: the tracked-set + // keys are borrowed by a hashmap that outlives the block that builds them, + // and a scoped `defer free` there dangles every key before the sweep reads + // it. One arena at function scope removes that hazard and the two smaller + // leaks around it. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var store = Store.init(io, allocator, ctx.config.cache_dir); + const keys = store.cacheKeys(arena) catch { + try out.print("Cache is empty; nothing to check.\n", .{}); + return; + }; + if (keys.len == 0) { + try out.print("Cache is empty; nothing to check.\n", .{}); + return; + } + + // The set zfin intends to keep fresh. Absent a portfolio every cached + // symbol counts as untracked, which is honest - without one there is + // nothing to be tracked BY. + var tracked = std.StringHashMap(void).init(arena); + if (cli.loadPortfolio(ctx, ctx.today)) |loaded_pf| { + var l = loaded_pf; + defer l.deinit(allocator); + const wl = ctx.resolveWatchlistPath(); + defer wl.deinit(allocator); + const wl_syms: ?[][]const u8 = if (ctx.globals.watchlist_path != null or wl.resolved != null) + cli.loadWatchlist(io, arena, wl.path) + else + null; + + const syms = try l.portfolio.fetchedSymbols(arena, .{ + .watchlist_syms = if (wl_syms) |w| w else &.{}, + .benchmarks = benchmarkPair(io, arena, l.anchor()), + }); + for (syms) |sym| try tracked.put(sym, {}); + } + + // Build the corpus. `readCandleMeta` returning null IS the filter for + // non-candle keys: EDGAR CIKs, negative-cached symbols, and anything never + // fetched all land here as `last_date = null`, and none of them can be + // reported stale. + var entries = std.ArrayList(freshness.Entry).empty; + for (keys) |key| { + const cm = store.readCandleMeta(key); + try entries.append(arena, .{ + .symbol = key, + .kind = zfin.market.classify(key), + .last_date = if (cm) |m| m.meta.last_date else null, + .tracked = tracked.contains(key), + }); + } + + var report = try freshness.scan(arena, entries.items, now_s); + defer report.deinit(arena); + + for (report.groups) |g| { + const label = switch (g.kind) { + .equity => "equity/ETF", + .mutual_fund => "mutual fund", + }; + if (g.peer_date) |pd| { + try out.print("{s:<12} {d} cached, newest bar {f}", .{ label, g.dated, pd }); + if (!g.conclusive()) { + try out.print(" - only one cached, no peer to compare against", .{}); + } else if (g.freshness) |f| switch (f) { + // Advice about the corpus, NOT a reason to withhold findings. + // The per-symbol comparison below stands on its own. + .lagging => try out.print(" - the latest session's bar is not published yet", .{}), + .overdue => try out.print(" - the whole group is a session or more behind; a refresh is due", .{}), + .current => {}, + }; + try out.print("\n", .{}); + } else { + try out.print("{s:<12} none cached\n", .{label}); + } + } + + if (report.stale.len > 0) { + try out.print("\nBehind their peers ({d}):\n", .{report.stale.len}); + for (report.stale) |f| { + try out.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date }); + } + try out.print("\n zfin --refresh-data=force portfolio re-fetches the tracked set\n", .{}); + } else { + try out.print("\nNothing behind its peers.\n", .{}); + } + + if (report.missing.len > 0) { + try out.print("\nTracked but never cached ({d}): ", .{report.missing.len}); + for (report.missing, 0..) |sym, i| try out.print("{s}{s}", .{ if (i == 0) "" else ", ", sym }); + try out.print("\n", .{}); + } + + if (report.orphans.len > 0) { + try out.print("\nCached but tracked by nothing ({d}): ", .{report.orphans.len}); + for (report.orphans, 0..) |sym, i| try out.print("{s}{s}", .{ if (i == 0) "" else ", ", sym }); + try out.print("\n Nothing will refresh these. `zfin cache clear` is the blunt option.\n", .{}); + } +} + +/// The projections benchmark pair, read from `projections.srf` beside the +/// portfolio. Returns the SPY/AGG defaults when the file is absent or silent. +/// +/// These belong in the tracked set even though they are held nowhere: they are +/// fetched for the benchmark comparison, which is why AGG went stale unnoticed +/// while SPY - which doubles as a `ticker::` alias on a real holding - stayed +/// current. +/// +/// `views/projections.zig` reads the same file for the same config, but keeps +/// its `UserConfig` alive and reads the fields in place. This exists only +/// because the tracked set outlives the config: an overridden symbol lives in a +/// `[16]u8` field inside `UserConfig`, so the slices must be copied out before +/// that struct dies. +fn benchmarkPair(io: std.Io, arena: std.mem.Allocator, anchor: []const u8) []const []const u8 { + const path = cli.siblingPath(arena, anchor, "projections.srf") catch return &.{}; + const data = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024)) catch null; + const cfg = projections.parseProjectionsConfig(data); + const pair = arena.alloc([]const u8, 2) catch return &.{}; + pair[0] = arena.dupe(u8, cfg.benchmark_stock) catch return &.{}; + pair[1] = arena.dupe(u8, cfg.benchmark_bond) catch return &.{}; + return pair; +} + fn runClear(ctx: *framework.RunCtx) !void { var store = Store.init(ctx.io, ctx.allocator, ctx.config.cache_dir); try store.clearAll(); @@ -392,3 +545,54 @@ test "formatSize: megabytes" { try std.testing.expectEqualStrings("1.0 MB", formatSize(&buf, 1024 * 1024)); try std.testing.expectEqualStrings("2.5 MB", formatSize(&buf, 2 * 1024 * 1024 + 512 * 1024)); } + +test "parseArgs: 'stale' resolves to .stale" { + var ctx: framework.RunCtx = undefined; + ctx.io = std.testing.io; + const args = [_][]const u8{"stale"}; + const parsed = try parseArgs(&ctx, &args); + try std.testing.expectEqual(Subcommand.stale, parsed.sub); +} + +test "benchmarkPair: defaults to SPY/AGG when projections.srf is absent" { + // These two must be in the tracked set even though they are held nowhere. + // AGG went stale unnoticed for exactly this reason - it is fetched only for + // the benchmark comparison, while SPY looked fine because it doubles as a + // `ticker::` alias on a real holding. + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const pair = benchmarkPair(std.testing.io, arena_state.allocator(), "/nonexistent/portfolio.srf"); + try std.testing.expectEqual(@as(usize, 2), pair.len); + try std.testing.expectEqualStrings("SPY", pair[0]); + try std.testing.expectEqualStrings("AGG", pair[1]); +} + +test "benchmarkPair: an override in projections.srf is honoured" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + { + const f = try tmp.dir.createFile(io, "projections.srf", .{}); + defer f.close(io); + var buf: [256]u8 = undefined; + var w = f.writer(io, &buf); + try w.interface.writeAll("#!srfv1\ntype::config,benchmark_stock::VTI,benchmark_bond::BND\n"); + try w.interface.flush(); + } + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const anchor = try std.fs.path.join(allocator, &.{ dir_path, "portfolio.srf" }); + defer allocator.free(anchor); + + // Duped out of the config's stack-local override buffers - borrowing them + // would dangle the moment benchmarkPair returned. + const pair = benchmarkPair(io, arena_state.allocator(), anchor); + try std.testing.expectEqual(@as(usize, 2), pair.len); + try std.testing.expectEqualStrings("VTI", pair[0]); + try std.testing.expectEqualStrings("BND", pair[1]); +} diff --git a/src/commands/common.zig b/src/commands/common.zig index 29a0f91..d2474af 100644 --- a/src/commands/common.zig +++ b/src/commands/common.zig @@ -389,6 +389,21 @@ fn printLoadSummaryImpl(io: std.Io, color: bool, s: LoadSummaryStats) !void { const portfolio_loader = @import("../portfolio_loader.zig"); +/// A path to `name` in the same directory as `anchor`. +/// +/// The portfolio's siblings - `accounts.srf`, `metadata.srf`, +/// `transaction_log.srf`, `projections.srf` - all live beside whichever +/// portfolio file was resolved, so "next to the anchor" is the one rule that +/// finds them regardless of `ZFIN_HOME`, `-p` patterns, or cwd. +/// +/// Shared rather than per-command: it was private to `doctor` and the second +/// caller promptly hand-rolled dirname + join, which is how the two would have +/// drifted on trailing-separator handling. +pub fn siblingPath(arena: std.mem.Allocator, anchor: []const u8, name: []const u8) ![]const u8 { + const dir_end = if (std.mem.lastIndexOfScalar(u8, anchor, std.fs.path.sep)) |idx| idx + 1 else 0; + return std.fmt.allocPrint(arena, "{s}{s}", .{ anchor[0..dir_end], name }); +} + pub const LoadedPortfolio = portfolio_loader.LoadedPortfolio; pub const PortfolioData = portfolio_loader.PortfolioData; pub const loadPortfolioFromConfig = portfolio_loader.loadPortfolioFromConfig; @@ -1475,3 +1490,12 @@ test "buildPortfolioData: builds summary + candle_map for stock positions" { try std.testing.expect(pf_data.summary.allocations.len > 0); try std.testing.expectApproxEqAbs(@as(f64, 20_000), pf_data.summary.total_value, 1.0); } + +test "siblingPath: joins a filename onto the anchor's directory" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + try std.testing.expectEqualStrings("/home/u/data/accounts.srf", try siblingPath(a, "/home/u/data/portfolio.srf", "accounts.srf")); + // Bare filename (no separator) -> sibling is just the name. + try std.testing.expectEqualStrings("accounts.srf", try siblingPath(a, "portfolio.srf", "accounts.srf")); +} diff --git a/src/commands/doctor.zig b/src/commands/doctor.zig index e1cdde8..07c10e4 100644 --- a/src/commands/doctor.zig +++ b/src/commands/doctor.zig @@ -457,11 +457,6 @@ fn checkSrfFile( } /// Join a sibling filename onto the anchor portfolio's directory. -fn siblingPath(arena: std.mem.Allocator, anchor: []const u8, name: []const u8) ![]const u8 { - const dir_end = if (std.mem.lastIndexOfScalar(u8, anchor, std.fs.path.sep)) |idx| idx + 1 else 0; - return std.fmt.allocPrint(arena, "{s}{s}", .{ anchor[0..dir_end], name }); -} - // ── run ─────────────────────────────────────────────────────── pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { @@ -505,10 +500,10 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { if (anchor) |a| { // Accounts - parsed + kept for cross-reference. { - const r = checkSrfFile(io, arena, "accounts.srf", try siblingPath(arena, a, "accounts.srf"), .optional, vAccounts); + const r = checkSrfFile(io, arena, "accounts.srf", try cli.siblingPath(arena, a, "accounts.srf"), .optional, vAccounts); try checks.append(arena, r); if (r.status == .ok) { - const path = try siblingPath(arena, a, "accounts.srf"); + const path = try cli.siblingPath(arena, a, "accounts.srf"); if (std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(16 * 1024 * 1024))) |b| { account_map = analysis.parseAccountsFile(arena, b) catch null; } else |_| {} @@ -516,10 +511,10 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { } // Metadata - parsed + kept. { - const r = checkSrfFile(io, arena, "metadata.srf", try siblingPath(arena, a, "metadata.srf"), .optional, vMetadata); + const r = checkSrfFile(io, arena, "metadata.srf", try cli.siblingPath(arena, a, "metadata.srf"), .optional, vMetadata); try checks.append(arena, r); if (r.status == .ok) { - const path = try siblingPath(arena, a, "metadata.srf"); + const path = try cli.siblingPath(arena, a, "metadata.srf"); if (std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(16 * 1024 * 1024))) |b| { class_map = classification.parseClassificationFile(arena, b) catch null; } else |_| {} @@ -527,16 +522,16 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { } // Transaction log - parsed + kept. { - const r = checkSrfFile(io, arena, "transaction_log.srf", try siblingPath(arena, a, "transaction_log.srf"), .optional, vTransfers); + const r = checkSrfFile(io, arena, "transaction_log.srf", try cli.siblingPath(arena, a, "transaction_log.srf"), .optional, vTransfers); try checks.append(arena, r); if (r.status == .ok) { - const path = try siblingPath(arena, a, "transaction_log.srf"); + const path = try cli.siblingPath(arena, a, "transaction_log.srf"); if (std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(16 * 1024 * 1024))) |b| { transfer_log = transaction_log.parseTransactionLogFile(arena, b) catch null; } else |_| {} } } - try checks.append(arena, checkSrfFile(io, arena, "projections.srf", try siblingPath(arena, a, "projections.srf"), .optional, validateSrf)); + try checks.append(arena, checkSrfFile(io, arena, "projections.srf", try cli.siblingPath(arena, a, "projections.srf"), .optional, validateSrf)); // imported_values.srf and the snapshots both live under // /history/, NOT directly beside the // portfolio file. @@ -1359,15 +1354,6 @@ test "cross-ref end to end: missing account surfaces as a warn" { try testing.expect(std.mem.indexOf(u8, c.detail, "Sample HSA") != null); } -test "siblingPath: joins a filename onto the anchor's directory" { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - const a = arena.allocator(); - try testing.expectEqualStrings("/home/u/data/accounts.srf", try siblingPath(a, "/home/u/data/portfolio.srf", "accounts.srf")); - // Bare filename (no separator) -> sibling is just the name. - try testing.expectEqualStrings("accounts.srf", try siblingPath(a, "portfolio.srf", "accounts.srf")); -} - test "validateSrf: accepts a valid stream, rejects a headerless one" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit();