diff --git a/src/cache/freshness.zig b/src/cache/freshness.zig index 395e3ab..7893855 100644 --- a/src/cache/freshness.zig +++ b/src/cache/freshness.zig @@ -34,6 +34,7 @@ const std = @import("std"); const Date = @import("../Date.zig"); const market = @import("../market.zig"); +const cache = @import("store.zig"); /// One cached symbol, as the caller found it. pub const Entry = struct { @@ -50,6 +51,26 @@ pub const Entry = struct { tracked: bool, }; +/// Days behind peers that ordinary lag can still explain. Beyond this, a finding +/// is reported as `far_behind` rather than `stale`. +/// +/// Four days, and it is deliberately tight: the worst legitimate gap is a Friday +/// bar missed ahead of a Monday holiday, which leaves a symbol four calendar days +/// behind peers holding Tuesday's. More than that is not lag. +/// +/// The split exists because the two have different answers. Inside the window a +/// refresh is the remedy. Outside it, a refresh has most likely already been +/// attempted and failed, so offering one as the fix sends the operator in +/// circles. +/// +/// This module does NOT infer a cause, and callers must not either. The +/// candidates are many and none of them are visible from here: the ticker +/// changed, the provider dropped coverage, that one symbol is failing auth or +/// being rate-limited at the tail of a fetch, the symbol form is wrong for the +/// provider, whatever refreshes this cache never included it in its set, or it +/// stopped trading. Report the gap and let the operator look. +pub const max_normal_lag_days: i64 = 4; + /// A tracked symbol whose newest bar is behind its peers'. pub const Finding = struct { symbol: []const u8, @@ -102,8 +123,13 @@ pub const GroupState = struct { }; pub const Report = struct { - /// Tracked symbols behind their peers, worst first. + /// Tracked symbols behind their peers by no more than + /// `max_normal_lag_days`, worst first. A refresh is the expected remedy. stale: []Finding, + /// Tracked symbols further behind than lag explains, worst first. Reported + /// separately because a refresh probably is not the answer - see + /// `max_normal_lag_days`. + far_behind: []Finding, /// Cached, dated, but tracked by nothing. Nothing will ever refresh these, /// so they are not staleness - they are disk you may want back. Reported /// separately precisely so they cannot dominate the stale list forever. @@ -115,12 +141,46 @@ pub const Report = struct { pub fn deinit(self: Report, allocator: std.mem.Allocator) void { allocator.free(self.stale); + allocator.free(self.far_behind); allocator.free(self.orphans); allocator.free(self.missing); allocator.free(self.groups); } }; +/// Read the cache and build the corpus `scan` consumes. +/// +/// The I/O half, deliberately separate from `scan`: reading is a loop over the +/// store, deciding is the part worth testing exhaustively, and keeping them +/// apart is what lets every branch of the decision be exercised with fixed +/// dates and no filesystem. +/// +/// `readCandleMeta` returning null IS the filter for non-candle keys. EDGAR +/// CIKs, negative-cached symbols and anything never fetched all arrive as +/// `last_date = null`, so none of them can be reported stale, and this function +/// needs to know nothing about what any of them are. +/// +/// Symbol strings borrow from `keys`; only the slice is allocated. +pub fn collect( + allocator: std.mem.Allocator, + store: *cache.Store, + keys: []const []const u8, + tracked: *const std.StringHashMap(void), +) ![]Entry { + var out = try allocator.alloc(Entry, keys.len); + errdefer allocator.free(out); + for (keys, 0..) |key, i| { + const cm = store.readCandleMeta(key); + out[i] = .{ + .symbol = key, + .kind = market.classify(key), + .last_date = if (cm) |m| m.meta.last_date else null, + .tracked = tracked.contains(key), + }; + } + return out; +} + /// Classify `entries` as of `now_s`. /// /// Symbol strings are borrowed from `entries`; only the slices are allocated. @@ -154,6 +214,8 @@ pub fn scan( var stale = std.ArrayList(Finding).empty; errdefer stale.deinit(allocator); + var far_behind = std.ArrayList(Finding).empty; + errdefer far_behind.deinit(allocator); var orphans = std.ArrayList([]const u8).empty; errdefer orphans.deinit(allocator); var missing = std.ArrayList([]const u8).empty; @@ -175,13 +237,19 @@ pub fn scan( const peer = g.peer_date.?; if (!last.lessThan(peer)) continue; - try stale.append(allocator, .{ + const days = @divTrunc(peer.toEpoch() - last.toEpoch(), std.time.s_per_day); + const finding = Finding{ .symbol = e.symbol, .kind = e.kind, .last_date = last, .peer_date = peer, - .days_behind = @divTrunc(peer.toEpoch() - last.toEpoch(), std.time.s_per_day), - }); + .days_behind = days, + }; + if (days > max_normal_lag_days) { + try far_behind.append(allocator, finding); + } else { + try stale.append(allocator, finding); + } } // Ownership moves out of the ArrayLists one at a time, so each completed @@ -193,6 +261,10 @@ pub fn scan( errdefer allocator.free(stale_out); std.mem.sort(Finding, stale_out, {}, Finding.worstFirst); + const far_out = try far_behind.toOwnedSlice(allocator); + errdefer allocator.free(far_out); + std.mem.sort(Finding, far_out, {}, Finding.worstFirst); + const orphans_out = try orphans.toOwnedSlice(allocator); errdefer allocator.free(orphans_out); @@ -200,6 +272,7 @@ pub fn scan( return .{ .stale = stale_out, + .far_behind = far_out, .orphans = orphans_out, .missing = missing_out, .groups = groups, @@ -380,15 +453,15 @@ test "scan: no candle meta means missing when tracked, ignored when not" { test "scan: findings are ordered worst first" { const a = testing.allocator; - // A symbol one session behind and one six weeks behind are different - // diagnoses; the second should not be buried under the first. + // One session behind and three days behind are different magnitudes; the + // larger should not be buried under the smaller. const fri = Date.fromYmd(2025, 6, 13); const entries = [_]Entry{ eq("AAPL", fri, true), eq("MSFT", fri, true), eq("NEAR", Date.fromYmd(2025, 6, 12), true), - eq("FAR", Date.fromYmd(2025, 5, 2), true), - eq("MID", Date.fromYmd(2025, 6, 6), true), + eq("MID", Date.fromYmd(2025, 6, 11), true), + eq("FAR", Date.fromYmd(2025, 6, 9), true), }; var r = try scan(a, &entries, fridayEvening()); defer r.deinit(a); @@ -397,7 +470,55 @@ test "scan: findings are ordered worst first" { try testing.expectEqualStrings("FAR", r.stale[0].symbol); try testing.expectEqualStrings("MID", r.stale[1].symbol); try testing.expectEqualStrings("NEAR", r.stale[2].symbol); - try testing.expectEqual(@as(i64, 42), r.stale[0].days_behind); + try testing.expectEqual(@as(i64, 4), r.stale[0].days_behind); + try testing.expectEqual(@as(usize, 0), r.far_behind.len); +} + +test "scan: a symbol far behind is separated from one merely lagging" { + const a = testing.allocator; + // Observed live: three symbols 3-4 days behind, which a refresh fixes, + // alongside one 42 days behind that no refresh had moved. In a single + // worst-first list the latter sits permanently on top and buries the + // actionable ones. + const fri = Date.fromYmd(2025, 6, 13); + const entries = [_]Entry{ + eq("AAPL", fri, true), + eq("MSFT", fri, true), + eq("LAGGY", Date.fromYmd(2025, 6, 10), true), + eq("STOPPED", Date.fromYmd(2025, 5, 2), true), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 1), r.stale.len); + try testing.expectEqualStrings("LAGGY", r.stale[0].symbol); + try testing.expectEqual(@as(usize, 1), r.far_behind.len); + try testing.expectEqualStrings("STOPPED", r.far_behind[0].symbol); + try testing.expectEqual(@as(i64, 42), r.far_behind[0].days_behind); +} + +test "scan: the boundary is at max_normal_lag_days, exclusive" { + const a = testing.allocator; + const peer = Date.fromYmd(2025, 6, 13); + // Exactly at the limit is still ordinary lag; one day past it is not. The + // limit is a Friday bar missed ahead of a Monday holiday. + const at = peer.addDays(@intCast(-max_normal_lag_days)); + const over = peer.addDays(@intCast(-(max_normal_lag_days + 1))); + const entries = [_]Entry{ + eq("AAPL", peer, true), + eq("MSFT", peer, true), + eq("AT", at, true), + eq("OVER", over, true), + }; + var r = try scan(a, &entries, fridayEvening()); + defer r.deinit(a); + + try testing.expectEqual(@as(usize, 1), r.stale.len); + try testing.expectEqualStrings("AT", r.stale[0].symbol); + try testing.expectEqual(@as(i64, 4), r.stale[0].days_behind); + try testing.expectEqual(@as(usize, 1), r.far_behind.len); + try testing.expectEqualStrings("OVER", r.far_behind[0].symbol); + try testing.expectEqual(@as(i64, 5), r.far_behind[0].days_behind); } test "scan: an empty corpus is not an error" { diff --git a/src/cache/store.zig b/src/cache/store.zig index 2206391..b32d3c6 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -275,6 +275,18 @@ pub const Store = struct { }; } + /// Is `name` a data cache key, or one of this store's own synthetic ones? + /// + /// `_edgar` holds the EDGAR ticker indexes and `_torn` archived torn-body + /// forensics. Both are created by this store and neither is a symbol. + /// + /// Shared by `cacheKeys` and `diskStats` because they disagreed: `doctor` + /// reported 41 symbols where `zfin cache stats` reported 40, the difference + /// being `_edgar`. + fn isDataKey(name: []const u8) bool { + return name.len > 0 and name[0] != '_'; + } + /// Aggregate on-disk cache statistics. pub const DiskStats = struct { symbols: usize = 0, files: usize = 0, bytes: u64 = 0 }; @@ -300,7 +312,9 @@ pub const Store = struct { stats.bytes += st.size; }, .directory => { - stats.symbols += 1; + // Synthetic keys are not symbols, but their bytes are still + // on the disk, so they count toward files/bytes below. + if (isDataKey(entry.name)) stats.symbols += 1; const subpath = std.fs.path.join(self.allocator, &.{ self.cache_dir, entry.name }) catch continue; defer self.allocator.free(subpath); var sub = std.Io.Dir.cwd().openDir(io, subpath, .{ .iterate = true }) catch continue; @@ -357,7 +371,7 @@ pub const Store = struct { var iter = dir.iterate(); while (iter.next(io) catch null) |entry| { if (entry.kind != .directory) continue; - if (entry.name.len == 0 or entry.name[0] == '_') continue; + if (!isDataKey(entry.name)) continue; const owned = try allocator.dupe(u8, entry.name); { errdefer allocator.free(owned); @@ -3879,3 +3893,29 @@ test "cacheKeys: every allocation-failure path unwinds cleanly" { var s = Store.init(io, allocator, dir_path); try std.testing.checkAllAllocationFailures(allocator, cacheKeysOom, .{&s}); } + +test "diskStats and cacheKeys agree on the symbol count" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + // They disagreed: `doctor` reported 41 symbols where `zfin cache stats` + // reported 40, because only one of them excluded `_edgar`. + for ([_][]const u8{ "AAPL", "NVDA", "_edgar", "_torn" }) |name| + try tmp.dir.createDir(io, name, std.Io.File.Permissions.default_dir); + (try tmp.dir.createFile(io, "cusip_tickers.srf", .{})).close(io); + + var s = Store.init(io, allocator, dir_path); + const keys = try s.cacheKeys(allocator); + defer Store.freeCacheKeys(allocator, keys); + const ds = s.diskStats(); + + try std.testing.expectEqual(keys.len, ds.symbols); + try std.testing.expectEqual(@as(usize, 2), ds.symbols); + // Synthetic keys are excluded from the symbol tally but their bytes are + // still on the disk, so the top-level file still counts toward `files`. + try std.testing.expectEqual(@as(usize, 1), ds.files); +} diff --git a/src/commands/cache.zig b/src/commands/cache.zig index e43e84d..9433462 100644 --- a/src/commands/cache.zig +++ b/src/commands/cache.zig @@ -4,7 +4,6 @@ const cli = @import("common.zig"); const framework = @import("framework.zig"); const srf = @import("srf"); const freshness = @import("../cache/freshness.zig"); -const projections = @import("../analytics/projections.zig"); const Store = zfin.cache.Store; const DataType = zfin.cache.DataType; @@ -244,36 +243,12 @@ fn runStale(ctx: *framework.RunCtx) !void { if (cli.loadPortfolio(ctx, ctx.today)) |loaded_pf| { var l = loaded_pf; defer l.deinit(allocator); - const wl = ctx.resolveWatchlistPath(); - defer wl.deinit(allocator); - const wl_syms: ?[][]const u8 = if (ctx.globals.watchlist_path != null or wl.resolved != null) - cli.loadWatchlist(io, arena, wl.path) - else - null; - - const syms = try l.portfolio.fetchedSymbols(arena, .{ - .watchlist_syms = if (wl_syms) |w| w else &.{}, - .benchmarks = benchmarkPair(io, arena, l.anchor()), - }); - for (syms) |sym| try tracked.put(sym, {}); + tracked = try cli.trackedSymbols(ctx, arena, l.portfolio, l.anchor()); } - // Build the corpus. `readCandleMeta` returning null IS the filter for - // non-candle keys: EDGAR CIKs, negative-cached symbols, and anything never - // fetched all land here as `last_date = null`, and none of them can be - // reported stale. - var entries = std.ArrayList(freshness.Entry).empty; - for (keys) |key| { - const cm = store.readCandleMeta(key); - try entries.append(arena, .{ - .symbol = key, - .kind = zfin.market.classify(key), - .last_date = if (cm) |m| m.meta.last_date else null, - .tracked = tracked.contains(key), - }); - } + const entries = try freshness.collect(arena, &store, keys, &tracked); - var report = try freshness.scan(arena, entries.items, now_s); + var report = try freshness.scan(arena, entries, now_s); defer report.deinit(arena); for (report.groups) |g| { @@ -304,10 +279,28 @@ fn runStale(ctx: *framework.RunCtx) !void { try out.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date }); } try out.print("\n zfin --refresh-data=force portfolio re-fetches the tracked set\n", .{}); - } else { + } else if (report.far_behind.len == 0) { try out.print("\nNothing behind its peers.\n", .{}); } + if (report.far_behind.len > 0) { + // Separated because a refresh is probably not the remedy. Naming no + // cause on purpose: none of the candidates is visible from here, and the + // first draft of this asserted "delisted", which is one of the least + // likely of them. + try out.print("\nFurther behind than lag explains ({d}):\n", .{report.far_behind.len}); + for (report.far_behind) |f| { + try out.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date }); + } + try out.print(" A refresh has probably already been tried on these. Worth checking:\n", .{}); + try out.print(" - whether whatever refreshes this cache includes the symbol at all\n", .{}); + try out.print(" (a shared server refreshes ITS symbol set, not yours)\n", .{}); + try out.print(" - whether the ticker changed, or the provider wants a different form\n", .{}); + try out.print(" - whether that one symbol is failing auth or being rate-limited\n", .{}); + try out.print(" - whether the provider still covers it, or it stopped trading\n", .{}); + try out.print(" `zfin cache stats` shows the per-symbol fetch state.\n", .{}); + } + if (report.missing.len > 0) { try out.print("\nTracked but never cached ({d}): ", .{report.missing.len}); for (report.missing, 0..) |sym, i| try out.print("{s}{s}", .{ if (i == 0) "" else ", ", sym }); @@ -321,29 +314,6 @@ fn runStale(ctx: *framework.RunCtx) !void { } } -/// The projections benchmark pair, read from `projections.srf` beside the -/// portfolio. Returns the SPY/AGG defaults when the file is absent or silent. -/// -/// These belong in the tracked set even though they are held nowhere: they are -/// fetched for the benchmark comparison, which is why AGG went stale unnoticed -/// while SPY - which doubles as a `ticker::` alias on a real holding - stayed -/// current. -/// -/// `views/projections.zig` reads the same file for the same config, but keeps -/// its `UserConfig` alive and reads the fields in place. This exists only -/// because the tracked set outlives the config: an overridden symbol lives in a -/// `[16]u8` field inside `UserConfig`, so the slices must be copied out before -/// that struct dies. -fn benchmarkPair(io: std.Io, arena: std.mem.Allocator, anchor: []const u8) []const []const u8 { - const path = cli.siblingPath(arena, anchor, "projections.srf") catch return &.{}; - const data = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024)) catch null; - const cfg = projections.parseProjectionsConfig(data); - const pair = arena.alloc([]const u8, 2) catch return &.{}; - pair[0] = arena.dupe(u8, cfg.benchmark_stock) catch return &.{}; - pair[1] = arena.dupe(u8, cfg.benchmark_bond) catch return &.{}; - return pair; -} - fn runClear(ctx: *framework.RunCtx) !void { var store = Store.init(ctx.io, ctx.allocator, ctx.config.cache_dir); try store.clearAll(); @@ -553,46 +523,3 @@ test "parseArgs: 'stale' resolves to .stale" { const parsed = try parseArgs(&ctx, &args); try std.testing.expectEqual(Subcommand.stale, parsed.sub); } - -test "benchmarkPair: defaults to SPY/AGG when projections.srf is absent" { - // These two must be in the tracked set even though they are held nowhere. - // AGG went stale unnoticed for exactly this reason - it is fetched only for - // the benchmark comparison, while SPY looked fine because it doubles as a - // `ticker::` alias on a real holding. - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - const pair = benchmarkPair(std.testing.io, arena_state.allocator(), "/nonexistent/portfolio.srf"); - try std.testing.expectEqual(@as(usize, 2), pair.len); - try std.testing.expectEqualStrings("SPY", pair[0]); - try std.testing.expectEqualStrings("AGG", pair[1]); -} - -test "benchmarkPair: an override in projections.srf is honoured" { - const allocator = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); - defer allocator.free(dir_path); - - { - const f = try tmp.dir.createFile(io, "projections.srf", .{}); - defer f.close(io); - var buf: [256]u8 = undefined; - var w = f.writer(io, &buf); - try w.interface.writeAll("#!srfv1\ntype::config,benchmark_stock::VTI,benchmark_bond::BND\n"); - try w.interface.flush(); - } - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const anchor = try std.fs.path.join(allocator, &.{ dir_path, "portfolio.srf" }); - defer allocator.free(anchor); - - // Duped out of the config's stack-local override buffers - borrowing them - // would dangle the moment benchmarkPair returned. - const pair = benchmarkPair(io, arena_state.allocator(), anchor); - try std.testing.expectEqual(@as(usize, 2), pair.len); - try std.testing.expectEqualStrings("VTI", pair[0]); - try std.testing.expectEqualStrings("BND", pair[1]); -} diff --git a/src/commands/common.zig b/src/commands/common.zig index d2474af..9425a7c 100644 --- a/src/commands/common.zig +++ b/src/commands/common.zig @@ -8,6 +8,7 @@ const framework = @import("framework.zig"); const stderr = @import("../stderr.zig"); pub const fmt = @import("../format.zig"); const theme = @import("../tui/theme.zig"); +const projections = @import("../analytics/projections.zig"); // ── Active CLI text palette ────────────────────────────────── // RGB foreground colors for ALL CLI (non-TUI) text output, emitted as @@ -389,6 +390,37 @@ fn printLoadSummaryImpl(io: std.Io, color: bool, s: LoadSummaryStats) !void { const portfolio_loader = @import("../portfolio_loader.zig"); +/// The set of symbols zfin intends to keep fresh, for a loaded portfolio. +/// +/// Holdings, `security_type::watch` lots, `watchlist.srf`, and the projections +/// benchmark pair - the union `Portfolio.fetchedSymbols` defines, with the +/// watchlist and benchmark inputs resolved from `ctx`. +/// +/// Keys borrow from `arena`; pass an arena that outlives every lookup. The +/// scoped-`defer`-frees version of this dangled its keys before the caller read +/// them, which made every cached symbol look untracked. +pub fn trackedSymbols( + ctx: *framework.RunCtx, + arena: std.mem.Allocator, + portfolio: zfin.Portfolio, + anchor: []const u8, +) !std.StringHashMap(void) { + var set = std.StringHashMap(void).init(arena); + const wl = ctx.resolveWatchlistPath(); + defer wl.deinit(ctx.allocator); + const wl_syms: ?[][]const u8 = if (ctx.globals.watchlist_path != null or wl.resolved != null) + loadWatchlist(ctx.io, arena, wl.path) + else + null; + + const syms = try portfolio.fetchedSymbols(arena, .{ + .watchlist_syms = if (wl_syms) |w| w else &.{}, + .benchmarks = benchmarkPair(ctx.io, arena, anchor), + }); + for (syms) |sym| try set.put(sym, {}); + return set; +} + /// A path to `name` in the same directory as `anchor`. /// /// The portfolio's siblings - `accounts.srf`, `metadata.srf`, @@ -404,6 +436,29 @@ pub fn siblingPath(arena: std.mem.Allocator, anchor: []const u8, name: []const u return std.fmt.allocPrint(arena, "{s}{s}", .{ anchor[0..dir_end], name }); } +/// The projections benchmark pair, read from `projections.srf` beside the +/// portfolio. Returns the SPY/AGG defaults when the file is absent or silent. +/// +/// These belong in the tracked set even though they are held nowhere: they are +/// fetched for the benchmark comparison, which is why AGG went stale unnoticed +/// while SPY - which doubles as a `ticker::` alias on a real holding - stayed +/// current. +/// +/// `views/projections.zig` reads the same file for the same config, but keeps +/// its `UserConfig` alive and reads the fields in place. This exists only +/// because the tracked set outlives the config: an overridden symbol lives in a +/// `[16]u8` field inside `UserConfig`, so the slices must be copied out before +/// that struct dies. +pub fn benchmarkPair(io: std.Io, arena: std.mem.Allocator, anchor: []const u8) []const []const u8 { + const path = siblingPath(arena, anchor, "projections.srf") catch return &.{}; + const data = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024)) catch null; + const cfg = projections.parseProjectionsConfig(data); + const pair = arena.alloc([]const u8, 2) catch return &.{}; + pair[0] = arena.dupe(u8, cfg.benchmark_stock) catch return &.{}; + pair[1] = arena.dupe(u8, cfg.benchmark_bond) catch return &.{}; + return pair; +} + pub const LoadedPortfolio = portfolio_loader.LoadedPortfolio; pub const PortfolioData = portfolio_loader.PortfolioData; pub const loadPortfolioFromConfig = portfolio_loader.loadPortfolioFromConfig; @@ -1499,3 +1554,46 @@ test "siblingPath: joins a filename onto the anchor's directory" { // Bare filename (no separator) -> sibling is just the name. try std.testing.expectEqualStrings("accounts.srf", try siblingPath(a, "portfolio.srf", "accounts.srf")); } + +test "benchmarkPair: defaults to SPY/AGG when projections.srf is absent" { + // These two must be in the tracked set even though they are held nowhere. + // AGG went stale unnoticed for exactly this reason - it is fetched only for + // the benchmark comparison, while SPY looked fine because it doubles as a + // `ticker::` alias on a real holding. + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const pair = benchmarkPair(std.testing.io, arena_state.allocator(), "/nonexistent/portfolio.srf"); + try std.testing.expectEqual(@as(usize, 2), pair.len); + try std.testing.expectEqualStrings("SPY", pair[0]); + try std.testing.expectEqualStrings("AGG", pair[1]); +} + +test "benchmarkPair: an override in projections.srf is honoured" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + { + const f = try tmp.dir.createFile(io, "projections.srf", .{}); + defer f.close(io); + var buf: [256]u8 = undefined; + var w = f.writer(io, &buf); + try w.interface.writeAll("#!srfv1\ntype::config,benchmark_stock::VTI,benchmark_bond::BND\n"); + try w.interface.flush(); + } + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const anchor = try std.fs.path.join(allocator, &.{ dir_path, "portfolio.srf" }); + defer allocator.free(anchor); + + // Duped out of the config's stack-local override buffers - borrowing them + // would dangle the moment benchmarkPair returned. + const pair = benchmarkPair(io, arena_state.allocator(), anchor); + try std.testing.expectEqual(@as(usize, 2), pair.len); + try std.testing.expectEqualStrings("VTI", pair[0]); + try std.testing.expectEqualStrings("BND", pair[1]); +} diff --git a/src/commands/doctor.zig b/src/commands/doctor.zig index 07c10e4..afc0078 100644 --- a/src/commands/doctor.zig +++ b/src/commands/doctor.zig @@ -34,6 +34,7 @@ const Config = zfin.Config; const Lot = @import("../models/portfolio.zig").Lot; const Date = @import("../Date.zig"); const cache = @import("../cache/store.zig"); +const freshness = @import("../cache/freshness.zig"); const classification = @import("../models/classification.zig"); const analysis = @import("../analytics/analysis.zig"); const transaction_log = @import("../models/transaction_log.zig"); @@ -160,6 +161,59 @@ fn joinCapped(arena: std.mem.Allocator, items: []const []const u8, cap: usize) ! return buf.items; } +/// Are any tracked symbols' candles behind their same-kind peers'? +/// +/// Reads the cache only - no fetches, no writes - which keeps `doctor`'s +/// read-only contract intact. `zfin cache stale` is the detailed view and +/// `zfin cache refresh` the acting one; this exists so the problem surfaces +/// without being asked about, since the failure it catches is silent by nature. +/// +/// Always `.warn` or better, never `.fail`: a `.fail` makes `doctor` exit +/// non-zero, and a stale cache is a thing to fix rather than a broken install. +/// Cron and CI stay green. +/// +/// Takes the tracked set and `now_s` rather than a `RunCtx` so it can be tested +/// against a temp cache directory. Building the tracked set is the caller's job. +fn checkCandleFreshness( + arena: std.mem.Allocator, + store: *cache.Store, + tracked: *const std.StringHashMap(void), + now_s: i64, +) !Check { + const label = "Candle freshness"; + const keys = store.cacheKeys(arena) catch + return .{ .status = .info, .label = label, .detail = "cache not readable" }; + if (keys.len == 0) return .{ .status = .info, .label = label, .detail = "nothing cached" }; + + const entries = try freshness.collect(arena, store, keys, tracked); + var report = try freshness.scan(arena, entries, now_s); + defer report.deinit(arena); + + if (report.stale.len == 0 and report.far_behind.len == 0) { + return .{ .status = .ok, .label = label, .detail = "no symbol is behind its peers" }; + } + + // Both buckets warn, but the detail keeps them apart: one is fixed by a + // refresh and the other probably is not. + var names: std.ArrayList([]const u8) = .empty; + for (report.stale) |f| { + try names.append(arena, try std.fmt.allocPrint(arena, "{s} ({d}d)", .{ f.symbol, f.days_behind })); + } + for (report.far_behind) |f| { + // No cause named - see `freshness.max_normal_lag_days`. "!" marks it as + // the bucket a refresh probably will not fix. + try names.append(arena, try std.fmt.allocPrint(arena, "{s} ({d}d!)", .{ f.symbol, f.days_behind })); + } + return .{ + .status = .warn, + .label = label, + .detail = try std.fmt.allocPrint(arena, "{d} behind peers: {s} - see `zfin cache stale`", .{ + report.stale.len + report.far_behind.len, + try joinCapped(arena, names.items, 6), + }), + }; +} + /// Cross-reference check: every name in `needed` should appear in /// `known`. OK when all are present (or `needed` is empty); WARN listing /// the missing ones otherwise. Operates on plain string slices so it's @@ -468,8 +522,13 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { var sections: std.ArrayList(Section) = .empty; - // Collected for cross-reference (Section B). + // Collected for cross-reference (Section B) and the cache checks + // (Section C), so both are hoisted out of Section A's block. var all_lots: std.ArrayList(Lot) = .empty; + // First resolved portfolio path. Names the directory the portfolio's + // siblings live in - `projections.srf` among them, which Section C needs + // for the benchmark pair. + var anchor: ?[]const u8 = null; var account_map: ?analysis.AccountMap = null; var class_map: ?classification.ClassificationMap = null; var transfer_log: ?transaction_log.TransactionLog = null; @@ -480,7 +539,6 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { const source: []const u8 = if (config.zfin_home) |h| h else "cwd"; // Portfolio file(s) - globbed, union-merged. Parse-check each. - var anchor: ?[]const u8 = null; const pf = config.resolveUserFiles(io, arena, Config.default_portfolio_filename) catch Config.ResolvedPaths{ .paths = &.{}, .allocator = arena }; if (pf.paths.len == 0) { @@ -605,6 +663,18 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void { .label = "Cache", .detail = try std.fmt.allocPrint(arena, "{d} symbols, {d} files, {s} ({s})", .{ ds.symbols, ds.files, cache_cmd.formatSize(&size_buf, ds.bytes), config.cache_dir }), }); + // Without a portfolio there is no tracked set, so every symbol + // would read as an orphan and nothing as stale - an empty set is + // the honest input. + var tracked = std.StringHashMap(void).init(arena); + if (anchor) |a| { + tracked = cli.trackedSymbols(ctx, arena, .{ .lots = all_lots.items, .allocator = arena }, a) catch + std.StringHashMap(void).init(arena); + } + // wall-clock required: peer freshness is judged against the market + // calendar, which needs the real instant rather than `ctx.today`. + const now_s = std.Io.Timestamp.now(io, .real).toSeconds(); + try checks.append(arena, try checkCandleFreshness(arena, &store, &tracked, now_s)); } // Hand-maintained data staleness. @@ -1425,3 +1495,119 @@ test "trimTrailingSlash: drops a single trailing slash" { try testing.expectEqualStrings("https://h", trimTrailingSlash("https://h/")); try testing.expectEqualStrings("https://h", trimTrailingSlash("https://h")); } + +/// Seed a cache directory with `candles_meta.srf` for each symbol/date pair. +fn seedCandleMeta(io: std.Io, tmp: *std.testing.TmpDir, pairs: []const struct { []const u8, []const u8 }) !void { + for (pairs) |p| { + try tmp.dir.createDir(io, p[0], std.Io.File.Permissions.default_dir); + const rel = try std.fmt.allocPrint(testing.allocator, "{s}/candles_meta.srf", .{p[0]}); + defer testing.allocator.free(rel); + const f = try tmp.dir.createFile(io, rel, .{}); + defer f.close(io); + var buf: [256]u8 = undefined; + var w = f.writer(io, &buf); + try w.interface.print( + "#!srfv1\n#!expires=99999999999\nlast_close:num:100.00,last_date::{s},provider::tiingo\n", + .{p[1]}, + ); + try w.interface.flush(); + } +} + +test "checkCandleFreshness: warns and names the laggards, never fails" { + const allocator = testing.allocator; + const io = testing.io; + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + try seedCandleMeta(io, &tmp, &.{ + .{ "AAPL", "2025-06-13" }, + .{ "MSFT", "2025-06-13" }, + .{ "NVDA", "2025-06-13" }, + .{ "TSLA", "2025-06-12" }, + }); + + var tracked = std.StringHashMap(void).init(arena); + for ([_][]const u8{ "AAPL", "MSFT", "NVDA", "TSLA" }) |s| try tracked.put(s, {}); + + var store = cache.Store.init(io, arena, dir_path); + // Friday 2025-06-13, 22:00 UTC - past the equity boundary. + const now_s = zfin.Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour; + const c = try checkCandleFreshness(arena, &store, &tracked, now_s); + + // WARN, not FAIL: `doctor` exits non-zero on any fail, and a stale cache is + // a thing to fix rather than a broken install. Cron stays green. + try testing.expectEqual(Status.warn, c.status); + try testing.expect(std.mem.indexOf(u8, c.detail, "TSLA (1d)") != null); + try testing.expect(std.mem.indexOf(u8, c.detail, "zfin cache stale") != null); + // The three current symbols are not named. + try testing.expect(std.mem.indexOf(u8, c.detail, "AAPL") == null); +} + +test "checkCandleFreshness: a caught-up corpus is OK" { + const allocator = testing.allocator; + const io = testing.io; + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + try seedCandleMeta(io, &tmp, &.{ .{ "AAPL", "2025-06-13" }, .{ "MSFT", "2025-06-13" } }); + + var tracked = std.StringHashMap(void).init(arena); + for ([_][]const u8{ "AAPL", "MSFT" }) |s| try tracked.put(s, {}); + + var store = cache.Store.init(io, arena, dir_path); + const now_s = zfin.Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour; + const c = try checkCandleFreshness(arena, &store, &tracked, now_s); + try testing.expectEqual(Status.ok, c.status); +} + +test "checkCandleFreshness: an empty cache is informational, not a warning" { + const allocator = testing.allocator; + const io = testing.io; + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + // First run on a fresh machine must not look like a problem. + var tracked = std.StringHashMap(void).init(arena); + var store = cache.Store.init(io, arena, "/nonexistent/zfin-cache"); + const c = try checkCandleFreshness(arena, &store, &tracked, 0); + try testing.expectEqual(Status.info, c.status); +} + +test "checkCandleFreshness: an untracked laggard does not warn" { + const allocator = testing.allocator; + const io = testing.io; + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + try seedCandleMeta(io, &tmp, &.{ + .{ "AAPL", "2025-06-13" }, + .{ "MSFT", "2025-06-13" }, + .{ "OLDCO", "2025-01-02" }, + }); + + // OLDCO is cached but tracked by nothing - an orphan. Nothing refreshes it, + // so warning about it every run would train the operator to skip the line. + var tracked = std.StringHashMap(void).init(arena); + for ([_][]const u8{ "AAPL", "MSFT" }) |s| try tracked.put(s, {}); + + var store = cache.Store.init(io, arena, dir_path); + const now_s = zfin.Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour; + const c = try checkCandleFreshness(arena, &store, &tracked, now_s); + try testing.expectEqual(Status.ok, c.status); +} diff --git a/src/service.zig b/src/service.zig index 70a2861..82c28f5 100644 --- a/src/service.zig +++ b/src/service.zig @@ -3141,6 +3141,39 @@ pub const DataService = struct { // Write to local cache var s = self.store(); + + // Never let the shared cache move a symbol BACKWARDS. The server's + // bytes are written verbatim, `#!expires=` included, so its view of + // freshness becomes the client's - and if the server's copy carries an + // older bar than the one already here, an unconditional write replaces + // good local data with worse and stamps it authoritative. + // + // Observed: the server's cron fetched at 17:00 ET, some symbols got that + // session's bar and some did not, and every one of them was stamped + // fresh until the next boundary. A client that had already fetched the + // newer bar would have had it overwritten and then believed the older + // one for a full day. + // + // Only candle data can be ordered this way, so only candle data is + // guarded; everything else falls through unchanged. + if (isCandleType(data_type)) { + if (serverBarRegression(&s, symbol, response.body)) |reg| { + // WARN, not debug. A shared cache should never be behind a + // client that draws from it: it is the tier with the refresh + // cron and the provider budget. When it happens, something on + // the server side has stopped keeping up, and every other client + // is being handed the same stale bar - so this is an operator + // event, not a diagnostic detail. Logged at debug, it was + // invisible in exactly the builds people install. + log.warn( + "{s}: shared cache is BEHIND this client for {s} - it offered {f}, local copy has {f}. Refused the sync rather than move the cache backwards; the server needs a refresh (see `zfin cache stale`).", + .{ symbol, @tagName(data_type), reg.incoming, reg.local }, + ); + log.debug("{s}: tryOneSync finished ({s}) result=ok elapsed_ms={d}", .{ symbol, @tagName(data_type), @divTrunc(std.Io.Timestamp.now(self.io, .awake).nanoseconds - t_start, std.time.ns_per_ms) }); + return .ok; + } + } + s.writeRaw(symbol, data_type, response.body) catch |err| { log.debug("{s}: failed to write synced {s} to cache: {s}", .{ symbol, @tagName(data_type), @errorName(err) }); log.debug("{s}: tryOneSync finished ({s}) result=net_err elapsed_ms={d}", .{ symbol, @tagName(data_type), @divTrunc(std.Io.Timestamp.now(self.io, .awake).nanoseconds - t_start, std.time.ns_per_ms) }); @@ -3152,6 +3185,54 @@ pub const DataService = struct { } /// Sync candle data (both daily and meta) from the server. + /// Do these bytes describe candle data, i.e. data with a newest-bar date + /// that can be compared for age? + fn isCandleType(data_type: cache.DataType) bool { + return data_type == .candles_daily or data_type == .candles_meta; + } + + /// Both dates, when writing `body` would replace the local candle data with + /// an OLDER bar. Null otherwise. + /// + /// Returns the pair rather than a bool because the caller has to report + /// them: "the shared cache is behind you" is only actionable with the two + /// dates attached. + /// + /// Null whenever the question cannot be answered - no local copy, an + /// unparseable body, no dates on either side - so an unknown never blocks a + /// sync. The guard only fires on a definite regression. + fn serverBarRegression( + s: *cache.Store, + symbol: []const u8, + body: []const u8, + ) ?struct { local: Date, incoming: Date } { + const local = s.readCandleMeta(symbol) orelse return null; + const incoming = newestDateIn(body) orelse return null; + if (!incoming.lessThan(local.meta.last_date)) return null; + return .{ .local = local.meta.last_date, .incoming = incoming }; + } + + /// Newest `last_date::` or `date::` value in an SRF body. + /// + /// Deliberately a scan for the maximum rather than a parse: `candles_meta` + /// carries one `last_date`, `candles_daily` carries a `date` per bar, and + /// the ordering of the latter is not something this check should assume. + fn newestDateIn(body: []const u8) ?Date { + var best: ?Date = null; + for ([_][]const u8{ "last_date::", "date::" }) |key| { + var rest = body; + while (std.mem.indexOf(u8, rest, key)) |idx| { + const start = idx + key.len; + rest = rest[start..]; + const end = std.mem.indexOfAny(u8, rest, ",\n\r") orelse rest.len; + if (Date.parse(rest[0..end])) |d| { + if (best == null or best.?.lessThan(d)) best = d; + } else |_| {} + } + } + return best; + } + fn syncCandlesFromServer(self: *DataService, symbol: []const u8) bool { const daily = self.syncFromServer(symbol, .candles_daily); const meta = self.syncFromServer(symbol, .candles_meta); @@ -4907,3 +4988,62 @@ test "earningsNeedsRefresh: chase window is inclusive at the boundary" { const past_window = [_]EarningsEvent{.{ .date = Date.fromYmd(2026, 6, 11), .estimate = 1.0 }}; try std.testing.expect(!DataService.earningsNeedsRefresh(&past_window, today, 14)); } + +test "newestDateIn: picks the maximum across candles_meta and candles_daily shapes" { + // `candles_meta` carries one `last_date`; `candles_daily` carries a `date` + // per bar, in an order this check must not assume. + const meta = "#!srfv1\n#!expires=1\nlast_close:num:100.00,last_date::2026-08-07,provider::tiingo\n"; + try std.testing.expect(DataService.newestDateIn(meta).?.eql(Date.fromYmd(2026, 8, 7))); + + // Deliberately out of order. + const daily = "#!srfv1\ndate::2026-08-05,close:num:1\ndate::2026-08-07,close:num:3\ndate::2026-08-06,close:num:2\n"; + try std.testing.expect(DataService.newestDateIn(daily).?.eql(Date.fromYmd(2026, 8, 7))); + + // Nothing parseable -> null, so the guard cannot fire on garbage. + try std.testing.expectEqual(@as(?Date, null), DataService.newestDateIn("#!srfv1\n")); + try std.testing.expectEqual(@as(?Date, null), DataService.newestDateIn("last_date::not-a-date\n")); +} + +test "isCandleType: only candle data is age-comparable" { + try std.testing.expect(DataService.isCandleType(.candles_daily)); + try std.testing.expect(DataService.isCandleType(.candles_meta)); + // Dividends, splits, options and the rest have no single newest-bar date, + // so they sync unguarded. + try std.testing.expect(!DataService.isCandleType(.dividends)); + try std.testing.expect(!DataService.isCandleType(.splits)); + try std.testing.expect(!DataService.isCandleType(.classification)); +} + +test "serverBarRegression: blocks a regression and reports both dates" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = cache.Store.init(io, allocator, dir_path); + // Local copy holds Monday's bar. + s.updateCandleMeta("AAPL", 100.0, Date.fromYmd(2026, 8, 10), .tiingo, 0, 9999999999); + + const older = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-07,provider::tiingo\n"; + const same = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-10,provider::tiingo\n"; + const newer = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-11,provider::tiingo\n"; + + // THE REGRESSION THIS GUARDS. The shared cache's cron fetched at 17:00 ET; + // some symbols got that session's bar and some did not, and all were + // stamped fresh until the next boundary. Written unconditionally, the older + // body replaces good local data and is then believed for a full day. + const reg = DataService.serverBarRegression(&s, "AAPL", older) orelse + return error.ExpectedRegression; + // Both dates come back, because the warning is only actionable with them. + try std.testing.expect(reg.local.eql(Date.fromYmd(2026, 8, 10))); + try std.testing.expect(reg.incoming.eql(Date.fromYmd(2026, 8, 7))); + + try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", same)); + try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", newer)); + + // Unknowns never block: no local copy, and an unparseable body. + try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "NOLOCAL", older)); + try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", "#!srfv1\n")); +}