lobo/zfin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

allow doctor to see stale cache items

Browse source
This commit is contained in:
Emil Lerch 2026-08-11 11:02:21 -07:00
parent 418068a3d1
commit a91e5b0495
Signed by: lobo
GPG key ID: A7B62D657EF764F8
6 changed files with 620 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
src/cache/freshness.zig vendored
View file

@ -34,6 +34,7 @@
const std = @import("std");
const Date = @import("../Date.zig");
const market = @import("../market.zig");
const cache = @import("store.zig");
/// One cached symbol, as the caller found it.
pub const Entry = struct {
@ -50,6 +51,26 @@ pub const Entry = struct {
tracked: bool,
};
/// Days behind peers that ordinary lag can still explain. Beyond this, a finding
/// is reported as `far_behind` rather than `stale`.
///
/// Four days, and it is deliberately tight: the worst legitimate gap is a Friday
/// bar missed ahead of a Monday holiday, which leaves a symbol four calendar days
/// behind peers holding Tuesday's. More than that is not lag.
///
/// The split exists because the two have different answers. Inside the window a
/// refresh is the remedy. Outside it, a refresh has most likely already been
/// attempted and failed, so offering one as the fix sends the operator in
/// circles.
///
/// This module does NOT infer a cause, and callers must not either. The
/// candidates are many and none of them are visible from here: the ticker
/// changed, the provider dropped coverage, that one symbol is failing auth or
/// being rate-limited at the tail of a fetch, the symbol form is wrong for the
/// provider, whatever refreshes this cache never included it in its set, or it
/// stopped trading. Report the gap and let the operator look.
pub const max_normal_lag_days: i64 = 4;
/// A tracked symbol whose newest bar is behind its peers'.
pub const Finding = struct {
symbol: []const u8,
@ -102,8 +123,13 @@ pub const GroupState = struct {
};
pub const Report = struct {
/// Tracked symbols behind their peers, worst first.
/// Tracked symbols behind their peers by no more than
/// `max_normal_lag_days`, worst first. A refresh is the expected remedy.
stale: []Finding,
/// Tracked symbols further behind than lag explains, worst first. Reported
/// separately because a refresh probably is not the answer - see
/// `max_normal_lag_days`.
far_behind: []Finding,
/// Cached, dated, but tracked by nothing. Nothing will ever refresh these,
/// so they are not staleness - they are disk you may want back. Reported
/// separately precisely so they cannot dominate the stale list forever.
@ -115,12 +141,46 @@ pub const Report = struct {
pub fn deinit(self: Report, allocator: std.mem.Allocator) void {
allocator.free(self.stale);
allocator.free(self.far_behind);
allocator.free(self.orphans);
allocator.free(self.missing);
allocator.free(self.groups);
}
};
/// Read the cache and build the corpus `scan` consumes.
///
/// The I/O half, deliberately separate from `scan`: reading is a loop over the
/// store, deciding is the part worth testing exhaustively, and keeping them
/// apart is what lets every branch of the decision be exercised with fixed
/// dates and no filesystem.
///
/// `readCandleMeta` returning null IS the filter for non-candle keys. EDGAR
/// CIKs, negative-cached symbols and anything never fetched all arrive as
/// `last_date = null`, so none of them can be reported stale, and this function
/// needs to know nothing about what any of them are.
///
/// Symbol strings borrow from `keys`; only the slice is allocated.
pub fn collect(
allocator: std.mem.Allocator,
store: *cache.Store,
keys: []const []const u8,
tracked: *const std.StringHashMap(void),
) ![]Entry {
var out = try allocator.alloc(Entry, keys.len);
errdefer allocator.free(out);
for (keys, 0..) |key, i| {
const cm = store.readCandleMeta(key);
out[i] = .{
.symbol = key,
.kind = market.classify(key),
.last_date = if (cm) |m| m.meta.last_date else null,
.tracked = tracked.contains(key),
};
}
return out;
}
/// Classify `entries` as of `now_s`.
///
/// Symbol strings are borrowed from `entries`; only the slices are allocated.
@ -154,6 +214,8 @@ pub fn scan(
var stale = std.ArrayList(Finding).empty;
errdefer stale.deinit(allocator);
var far_behind = std.ArrayList(Finding).empty;
errdefer far_behind.deinit(allocator);
var orphans = std.ArrayList([]const u8).empty;
errdefer orphans.deinit(allocator);
var missing = std.ArrayList([]const u8).empty;
@ -175,13 +237,19 @@ pub fn scan(
const peer = g.peer_date.?;
if (!last.lessThan(peer)) continue;
try stale.append(allocator, .{
const days = @divTrunc(peer.toEpoch() - last.toEpoch(), std.time.s_per_day);
const finding = Finding{
.symbol = e.symbol,
.kind = e.kind,
.last_date = last,
.peer_date = peer,
.days_behind = @divTrunc(peer.toEpoch() - last.toEpoch(), std.time.s_per_day),
});
.days_behind = days,
};
if (days > max_normal_lag_days) {
try far_behind.append(allocator, finding);
} else {
try stale.append(allocator, finding);
}
}
// Ownership moves out of the ArrayLists one at a time, so each completed
@ -193,6 +261,10 @@ pub fn scan(
errdefer allocator.free(stale_out);
std.mem.sort(Finding, stale_out, {}, Finding.worstFirst);
const far_out = try far_behind.toOwnedSlice(allocator);
errdefer allocator.free(far_out);
std.mem.sort(Finding, far_out, {}, Finding.worstFirst);
const orphans_out = try orphans.toOwnedSlice(allocator);
errdefer allocator.free(orphans_out);
@ -200,6 +272,7 @@ pub fn scan(
return .{
.stale = stale_out,
.far_behind = far_out,
.orphans = orphans_out,
.missing = missing_out,
.groups = groups,
@ -380,15 +453,15 @@ test "scan: no candle meta means missing when tracked, ignored when not" {
test "scan: findings are ordered worst first" {
const a = testing.allocator;
// A symbol one session behind and one six weeks behind are different
// diagnoses; the second should not be buried under the first.
// One session behind and three days behind are different magnitudes; the
// larger should not be buried under the smaller.
const fri = Date.fromYmd(2025, 6, 13);
const entries = [_]Entry{
eq("AAPL", fri, true),
eq("MSFT", fri, true),
eq("NEAR", Date.fromYmd(2025, 6, 12), true),
eq("FAR", Date.fromYmd(2025, 5, 2), true),
eq("MID", Date.fromYmd(2025, 6, 6), true),
eq("MID", Date.fromYmd(2025, 6, 11), true),
eq("FAR", Date.fromYmd(2025, 6, 9), true),
};
var r = try scan(a, &entries, fridayEvening());
defer r.deinit(a);
@ -397,7 +470,55 @@ test "scan: findings are ordered worst first" {
try testing.expectEqualStrings("FAR", r.stale[0].symbol);
try testing.expectEqualStrings("MID", r.stale[1].symbol);
try testing.expectEqualStrings("NEAR", r.stale[2].symbol);
try testing.expectEqual(@as(i64, 42), r.stale[0].days_behind);
try testing.expectEqual(@as(i64, 4), r.stale[0].days_behind);
try testing.expectEqual(@as(usize, 0), r.far_behind.len);
}
test "scan: a symbol far behind is separated from one merely lagging" {
const a = testing.allocator;
// Observed live: three symbols 3-4 days behind, which a refresh fixes,
// alongside one 42 days behind that no refresh had moved. In a single
// worst-first list the latter sits permanently on top and buries the
// actionable ones.
const fri = Date.fromYmd(2025, 6, 13);
const entries = [_]Entry{
eq("AAPL", fri, true),
eq("MSFT", fri, true),
eq("LAGGY", Date.fromYmd(2025, 6, 10), true),
eq("STOPPED", Date.fromYmd(2025, 5, 2), true),
};
var r = try scan(a, &entries, fridayEvening());
defer r.deinit(a);
try testing.expectEqual(@as(usize, 1), r.stale.len);
try testing.expectEqualStrings("LAGGY", r.stale[0].symbol);
try testing.expectEqual(@as(usize, 1), r.far_behind.len);
try testing.expectEqualStrings("STOPPED", r.far_behind[0].symbol);
try testing.expectEqual(@as(i64, 42), r.far_behind[0].days_behind);
}
test "scan: the boundary is at max_normal_lag_days, exclusive" {
const a = testing.allocator;
const peer = Date.fromYmd(2025, 6, 13);
// Exactly at the limit is still ordinary lag; one day past it is not. The
// limit is a Friday bar missed ahead of a Monday holiday.
const at = peer.addDays(@intCast(-max_normal_lag_days));
const over = peer.addDays(@intCast(-(max_normal_lag_days + 1)));
const entries = [_]Entry{
eq("AAPL", peer, true),
eq("MSFT", peer, true),
eq("AT", at, true),
eq("OVER", over, true),
};
var r = try scan(a, &entries, fridayEvening());
defer r.deinit(a);
try testing.expectEqual(@as(usize, 1), r.stale.len);
try testing.expectEqualStrings("AT", r.stale[0].symbol);
try testing.expectEqual(@as(i64, 4), r.stale[0].days_behind);
try testing.expectEqual(@as(usize, 1), r.far_behind.len);
try testing.expectEqualStrings("OVER", r.far_behind[0].symbol);
try testing.expectEqual(@as(i64, 5), r.far_behind[0].days_behind);
}
test "scan: an empty corpus is not an error" {

44
src/cache/store.zig vendored
View file

@ -275,6 +275,18 @@ pub const Store = struct {
};
}
/// Is `name` a data cache key, or one of this store's own synthetic ones?
///
/// `_edgar` holds the EDGAR ticker indexes and `_torn` archived torn-body
/// forensics. Both are created by this store and neither is a symbol.
///
/// Shared by `cacheKeys` and `diskStats` because they disagreed: `doctor`
/// reported 41 symbols where `zfin cache stats` reported 40, the difference
/// being `_edgar`.
fn isDataKey(name: []const u8) bool {
return name.len > 0 and name[0] != '_';
}
/// Aggregate on-disk cache statistics.
pub const DiskStats = struct { symbols: usize = 0, files: usize = 0, bytes: u64 = 0 };
@ -300,7 +312,9 @@ pub const Store = struct {
stats.bytes += st.size;
},
.directory => {
stats.symbols += 1;
// Synthetic keys are not symbols, but their bytes are still
// on the disk, so they count toward files/bytes below.
if (isDataKey(entry.name)) stats.symbols += 1;
const subpath = std.fs.path.join(self.allocator, &.{ self.cache_dir, entry.name }) catch continue;
defer self.allocator.free(subpath);
var sub = std.Io.Dir.cwd().openDir(io, subpath, .{ .iterate = true }) catch continue;
@ -357,7 +371,7 @@ pub const Store = struct {
var iter = dir.iterate();
while (iter.next(io) catch null) |entry| {
if (entry.kind != .directory) continue;
if (entry.name.len == 0 or entry.name[0] == '_') continue;
if (!isDataKey(entry.name)) continue;
const owned = try allocator.dupe(u8, entry.name);
{
errdefer allocator.free(owned);
@ -3879,3 +3893,29 @@ test "cacheKeys: every allocation-failure path unwinds cleanly" {
var s = Store.init(io, allocator, dir_path);
try std.testing.checkAllAllocationFailures(allocator, cacheKeysOom, .{&s});
}
test "diskStats and cacheKeys agree on the symbol count" {
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
// They disagreed: `doctor` reported 41 symbols where `zfin cache stats`
// reported 40, because only one of them excluded `_edgar`.
for ([_][]const u8{ "AAPL", "NVDA", "_edgar", "_torn" }) |name|
try tmp.dir.createDir(io, name, std.Io.File.Permissions.default_dir);
(try tmp.dir.createFile(io, "cusip_tickers.srf", .{})).close(io);
var s = Store.init(io, allocator, dir_path);
const keys = try s.cacheKeys(allocator);
defer Store.freeCacheKeys(allocator, keys);
const ds = s.diskStats();
try std.testing.expectEqual(keys.len, ds.symbols);
try std.testing.expectEqual(@as(usize, 2), ds.symbols);
// Synthetic keys are excluded from the symbol tally but their bytes are
// still on the disk, so the top-level file still counts toward `files`.
try std.testing.expectEqual(@as(usize, 1), ds.files);
}

117
src/commands/cache.zig
View file

@ -4,7 +4,6 @@ const cli = @import("common.zig");
const framework = @import("framework.zig");
const srf = @import("srf");
const freshness = @import("../cache/freshness.zig");
const projections = @import("../analytics/projections.zig");
const Store = zfin.cache.Store;
const DataType = zfin.cache.DataType;
@ -244,36 +243,12 @@ fn runStale(ctx: *framework.RunCtx) !void {
if (cli.loadPortfolio(ctx, ctx.today)) |loaded_pf| {
var l = loaded_pf;
defer l.deinit(allocator);
const wl = ctx.resolveWatchlistPath();
defer wl.deinit(allocator);
const wl_syms: ?[][]const u8 = if (ctx.globals.watchlist_path != null or wl.resolved != null)
cli.loadWatchlist(io, arena, wl.path)
else
null;
const syms = try l.portfolio.fetchedSymbols(arena, .{
.watchlist_syms = if (wl_syms) |w| w else &.{},
.benchmarks = benchmarkPair(io, arena, l.anchor()),
});
for (syms) |sym| try tracked.put(sym, {});
tracked = try cli.trackedSymbols(ctx, arena, l.portfolio, l.anchor());
}
// Build the corpus. `readCandleMeta` returning null IS the filter for
// non-candle keys: EDGAR CIKs, negative-cached symbols, and anything never
// fetched all land here as `last_date = null`, and none of them can be
// reported stale.
var entries = std.ArrayList(freshness.Entry).empty;
for (keys) |key| {
const cm = store.readCandleMeta(key);
try entries.append(arena, .{
.symbol = key,
.kind = zfin.market.classify(key),
.last_date = if (cm) |m| m.meta.last_date else null,
.tracked = tracked.contains(key),
});
}
const entries = try freshness.collect(arena, &store, keys, &tracked);
var report = try freshness.scan(arena, entries.items, now_s);
var report = try freshness.scan(arena, entries, now_s);
defer report.deinit(arena);
for (report.groups) |g| {
@ -304,10 +279,28 @@ fn runStale(ctx: *framework.RunCtx) !void {
try out.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date });
}
try out.print("\n zfin --refresh-data=force portfolio re-fetches the tracked set\n", .{});
} else {
} else if (report.far_behind.len == 0) {
try out.print("\nNothing behind its peers.\n", .{});
}
if (report.far_behind.len > 0) {
// Separated because a refresh is probably not the remedy. Naming no
// cause on purpose: none of the candidates is visible from here, and the
// first draft of this asserted "delisted", which is one of the least
// likely of them.
try out.print("\nFurther behind than lag explains ({d}):\n", .{report.far_behind.len});
for (report.far_behind) |f| {
try out.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date });
}
try out.print(" A refresh has probably already been tried on these. Worth checking:\n", .{});
try out.print(" - whether whatever refreshes this cache includes the symbol at all\n", .{});
try out.print(" (a shared server refreshes ITS symbol set, not yours)\n", .{});
try out.print(" - whether the ticker changed, or the provider wants a different form\n", .{});
try out.print(" - whether that one symbol is failing auth or being rate-limited\n", .{});
try out.print(" - whether the provider still covers it, or it stopped trading\n", .{});
try out.print(" `zfin cache stats` shows the per-symbol fetch state.\n", .{});
}
if (report.missing.len > 0) {
try out.print("\nTracked but never cached ({d}): ", .{report.missing.len});
for (report.missing, 0..) |sym, i| try out.print("{s}{s}", .{ if (i == 0) "" else ", ", sym });
@ -321,29 +314,6 @@ fn runStale(ctx: *framework.RunCtx) !void {
}
}
/// The projections benchmark pair, read from `projections.srf` beside the
/// portfolio. Returns the SPY/AGG defaults when the file is absent or silent.
///
/// These belong in the tracked set even though they are held nowhere: they are
/// fetched for the benchmark comparison, which is why AGG went stale unnoticed
/// while SPY - which doubles as a `ticker::` alias on a real holding - stayed
/// current.
///
/// `views/projections.zig` reads the same file for the same config, but keeps
/// its `UserConfig` alive and reads the fields in place. This exists only
/// because the tracked set outlives the config: an overridden symbol lives in a
/// `[16]u8` field inside `UserConfig`, so the slices must be copied out before
/// that struct dies.
fn benchmarkPair(io: std.Io, arena: std.mem.Allocator, anchor: []const u8) []const []const u8 {
const path = cli.siblingPath(arena, anchor, "projections.srf") catch return &.{};
const data = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024)) catch null;
const cfg = projections.parseProjectionsConfig(data);
const pair = arena.alloc([]const u8, 2) catch return &.{};
pair[0] = arena.dupe(u8, cfg.benchmark_stock) catch return &.{};
pair[1] = arena.dupe(u8, cfg.benchmark_bond) catch return &.{};
return pair;
}
fn runClear(ctx: *framework.RunCtx) !void {
var store = Store.init(ctx.io, ctx.allocator, ctx.config.cache_dir);
try store.clearAll();
@ -553,46 +523,3 @@ test "parseArgs: 'stale' resolves to .stale" {
const parsed = try parseArgs(&ctx, &args);
try std.testing.expectEqual(Subcommand.stale, parsed.sub);
}
test "benchmarkPair: defaults to SPY/AGG when projections.srf is absent" {
// These two must be in the tracked set even though they are held nowhere.
// AGG went stale unnoticed for exactly this reason - it is fetched only for
// the benchmark comparison, while SPY looked fine because it doubles as a
// `ticker::` alias on a real holding.
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const pair = benchmarkPair(std.testing.io, arena_state.allocator(), "/nonexistent/portfolio.srf");
try std.testing.expectEqual(@as(usize, 2), pair.len);
try std.testing.expectEqualStrings("SPY", pair[0]);
try std.testing.expectEqualStrings("AGG", pair[1]);
}
test "benchmarkPair: an override in projections.srf is honoured" {
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
{
const f = try tmp.dir.createFile(io, "projections.srf", .{});
defer f.close(io);
var buf: [256]u8 = undefined;
var w = f.writer(io, &buf);
try w.interface.writeAll("#!srfv1\ntype::config,benchmark_stock::VTI,benchmark_bond::BND\n");
try w.interface.flush();
}
var arena_state = std.heap.ArenaAllocator.init(allocator);
defer arena_state.deinit();
const anchor = try std.fs.path.join(allocator, &.{ dir_path, "portfolio.srf" });
defer allocator.free(anchor);
// Duped out of the config's stack-local override buffers - borrowing them
// would dangle the moment benchmarkPair returned.
const pair = benchmarkPair(io, arena_state.allocator(), anchor);
try std.testing.expectEqual(@as(usize, 2), pair.len);
try std.testing.expectEqualStrings("VTI", pair[0]);
try std.testing.expectEqualStrings("BND", pair[1]);
}

98
src/commands/common.zig
View file

@ -8,6 +8,7 @@ const framework = @import("framework.zig");
const stderr = @import("../stderr.zig");
pub const fmt = @import("../format.zig");
const theme = @import("../tui/theme.zig");
const projections = @import("../analytics/projections.zig");
// ── Active CLI text palette ──────────────────────────────────
// RGB foreground colors for ALL CLI (non-TUI) text output, emitted as
@ -389,6 +390,37 @@ fn printLoadSummaryImpl(io: std.Io, color: bool, s: LoadSummaryStats) !void {
const portfolio_loader = @import("../portfolio_loader.zig");
/// The set of symbols zfin intends to keep fresh, for a loaded portfolio.
///
/// Holdings, `security_type::watch` lots, `watchlist.srf`, and the projections
/// benchmark pair - the union `Portfolio.fetchedSymbols` defines, with the
/// watchlist and benchmark inputs resolved from `ctx`.
///
/// Keys borrow from `arena`; pass an arena that outlives every lookup. The
/// scoped-`defer`-frees version of this dangled its keys before the caller read
/// them, which made every cached symbol look untracked.
pub fn trackedSymbols(
ctx: *framework.RunCtx,
arena: std.mem.Allocator,
portfolio: zfin.Portfolio,
anchor: []const u8,
) !std.StringHashMap(void) {
var set = std.StringHashMap(void).init(arena);
const wl = ctx.resolveWatchlistPath();
defer wl.deinit(ctx.allocator);
const wl_syms: ?[][]const u8 = if (ctx.globals.watchlist_path != null or wl.resolved != null)
loadWatchlist(ctx.io, arena, wl.path)
else
null;
const syms = try portfolio.fetchedSymbols(arena, .{
.watchlist_syms = if (wl_syms) |w| w else &.{},
.benchmarks = benchmarkPair(ctx.io, arena, anchor),
});
for (syms) |sym| try set.put(sym, {});
return set;
}
/// A path to `name` in the same directory as `anchor`.
///
/// The portfolio's siblings - `accounts.srf`, `metadata.srf`,
@ -404,6 +436,29 @@ pub fn siblingPath(arena: std.mem.Allocator, anchor: []const u8, name: []const u
return std.fmt.allocPrint(arena, "{s}{s}", .{ anchor[0..dir_end], name });
}
/// The projections benchmark pair, read from `projections.srf` beside the
/// portfolio. Returns the SPY/AGG defaults when the file is absent or silent.
///
/// These belong in the tracked set even though they are held nowhere: they are
/// fetched for the benchmark comparison, which is why AGG went stale unnoticed
/// while SPY - which doubles as a `ticker::` alias on a real holding - stayed
/// current.
///
/// `views/projections.zig` reads the same file for the same config, but keeps
/// its `UserConfig` alive and reads the fields in place. This exists only
/// because the tracked set outlives the config: an overridden symbol lives in a
/// `[16]u8` field inside `UserConfig`, so the slices must be copied out before
/// that struct dies.
pub fn benchmarkPair(io: std.Io, arena: std.mem.Allocator, anchor: []const u8) []const []const u8 {
const path = siblingPath(arena, anchor, "projections.srf") catch return &.{};
const data = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024)) catch null;
const cfg = projections.parseProjectionsConfig(data);
const pair = arena.alloc([]const u8, 2) catch return &.{};
pair[0] = arena.dupe(u8, cfg.benchmark_stock) catch return &.{};
pair[1] = arena.dupe(u8, cfg.benchmark_bond) catch return &.{};
return pair;
}
pub const LoadedPortfolio = portfolio_loader.LoadedPortfolio;
pub const PortfolioData = portfolio_loader.PortfolioData;
pub const loadPortfolioFromConfig = portfolio_loader.loadPortfolioFromConfig;
@ -1499,3 +1554,46 @@ test "siblingPath: joins a filename onto the anchor's directory" {
// Bare filename (no separator) -> sibling is just the name.
try std.testing.expectEqualStrings("accounts.srf", try siblingPath(a, "portfolio.srf", "accounts.srf"));
}
test "benchmarkPair: defaults to SPY/AGG when projections.srf is absent" {
// These two must be in the tracked set even though they are held nowhere.
// AGG went stale unnoticed for exactly this reason - it is fetched only for
// the benchmark comparison, while SPY looked fine because it doubles as a
// `ticker::` alias on a real holding.
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const pair = benchmarkPair(std.testing.io, arena_state.allocator(), "/nonexistent/portfolio.srf");
try std.testing.expectEqual(@as(usize, 2), pair.len);
try std.testing.expectEqualStrings("SPY", pair[0]);
try std.testing.expectEqualStrings("AGG", pair[1]);
}
test "benchmarkPair: an override in projections.srf is honoured" {
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
{
const f = try tmp.dir.createFile(io, "projections.srf", .{});
defer f.close(io);
var buf: [256]u8 = undefined;
var w = f.writer(io, &buf);
try w.interface.writeAll("#!srfv1\ntype::config,benchmark_stock::VTI,benchmark_bond::BND\n");
try w.interface.flush();
}
var arena_state = std.heap.ArenaAllocator.init(allocator);
defer arena_state.deinit();
const anchor = try std.fs.path.join(allocator, &.{ dir_path, "portfolio.srf" });
defer allocator.free(anchor);
// Duped out of the config's stack-local override buffers - borrowing them
// would dangle the moment benchmarkPair returned.
const pair = benchmarkPair(io, arena_state.allocator(), anchor);
try std.testing.expectEqual(@as(usize, 2), pair.len);
try std.testing.expectEqualStrings("VTI", pair[0]);
try std.testing.expectEqualStrings("BND", pair[1]);
}

190
src/commands/doctor.zig
View file

@ -34,6 +34,7 @@ const Config = zfin.Config;
const Lot = @import("../models/portfolio.zig").Lot;
const Date = @import("../Date.zig");
const cache = @import("../cache/store.zig");
const freshness = @import("../cache/freshness.zig");
const classification = @import("../models/classification.zig");
const analysis = @import("../analytics/analysis.zig");
const transaction_log = @import("../models/transaction_log.zig");
@ -160,6 +161,59 @@ fn joinCapped(arena: std.mem.Allocator, items: []const []const u8, cap: usize) !
return buf.items;
}
/// Are any tracked symbols' candles behind their same-kind peers'?
///
/// Reads the cache only - no fetches, no writes - which keeps `doctor`'s
/// read-only contract intact. `zfin cache stale` is the detailed view and
/// `zfin cache refresh` the acting one; this exists so the problem surfaces
/// without being asked about, since the failure it catches is silent by nature.
///
/// Always `.warn` or better, never `.fail`: a `.fail` makes `doctor` exit
/// non-zero, and a stale cache is a thing to fix rather than a broken install.
/// Cron and CI stay green.
///
/// Takes the tracked set and `now_s` rather than a `RunCtx` so it can be tested
/// against a temp cache directory. Building the tracked set is the caller's job.
fn checkCandleFreshness(
arena: std.mem.Allocator,
store: *cache.Store,
tracked: *const std.StringHashMap(void),
now_s: i64,
) !Check {
const label = "Candle freshness";
const keys = store.cacheKeys(arena) catch
return .{ .status = .info, .label = label, .detail = "cache not readable" };
if (keys.len == 0) return .{ .status = .info, .label = label, .detail = "nothing cached" };
const entries = try freshness.collect(arena, store, keys, tracked);
var report = try freshness.scan(arena, entries, now_s);
defer report.deinit(arena);
if (report.stale.len == 0 and report.far_behind.len == 0) {
return .{ .status = .ok, .label = label, .detail = "no symbol is behind its peers" };
}
// Both buckets warn, but the detail keeps them apart: one is fixed by a
// refresh and the other probably is not.
var names: std.ArrayList([]const u8) = .empty;
for (report.stale) |f| {
try names.append(arena, try std.fmt.allocPrint(arena, "{s} ({d}d)", .{ f.symbol, f.days_behind }));
}
for (report.far_behind) |f| {
// No cause named - see `freshness.max_normal_lag_days`. "!" marks it as
// the bucket a refresh probably will not fix.
try names.append(arena, try std.fmt.allocPrint(arena, "{s} ({d}d!)", .{ f.symbol, f.days_behind }));
}
return .{
.status = .warn,
.label = label,
.detail = try std.fmt.allocPrint(arena, "{d} behind peers: {s} - see `zfin cache stale`", .{
report.stale.len + report.far_behind.len,
try joinCapped(arena, names.items, 6),
}),
};
}
/// Cross-reference check: every name in `needed` should appear in
/// `known`. OK when all are present (or `needed` is empty); WARN listing
/// the missing ones otherwise. Operates on plain string slices so it's
@ -468,8 +522,13 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
var sections: std.ArrayList(Section) = .empty;
// Collected for cross-reference (Section B).
// Collected for cross-reference (Section B) and the cache checks
// (Section C), so both are hoisted out of Section A's block.
var all_lots: std.ArrayList(Lot) = .empty;
// First resolved portfolio path. Names the directory the portfolio's
// siblings live in - `projections.srf` among them, which Section C needs
// for the benchmark pair.
var anchor: ?[]const u8 = null;
var account_map: ?analysis.AccountMap = null;
var class_map: ?classification.ClassificationMap = null;
var transfer_log: ?transaction_log.TransactionLog = null;
@ -480,7 +539,6 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
const source: []const u8 = if (config.zfin_home) |h| h else "cwd";
// Portfolio file(s) - globbed, union-merged. Parse-check each.
var anchor: ?[]const u8 = null;
const pf = config.resolveUserFiles(io, arena, Config.default_portfolio_filename) catch
Config.ResolvedPaths{ .paths = &.{}, .allocator = arena };
if (pf.paths.len == 0) {
@ -605,6 +663,18 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
.label = "Cache",
.detail = try std.fmt.allocPrint(arena, "{d} symbols, {d} files, {s} ({s})", .{ ds.symbols, ds.files, cache_cmd.formatSize(&size_buf, ds.bytes), config.cache_dir }),
});
// Without a portfolio there is no tracked set, so every symbol
// would read as an orphan and nothing as stale - an empty set is
// the honest input.
var tracked = std.StringHashMap(void).init(arena);
if (anchor) |a| {
tracked = cli.trackedSymbols(ctx, arena, .{ .lots = all_lots.items, .allocator = arena }, a) catch
std.StringHashMap(void).init(arena);
}
// wall-clock required: peer freshness is judged against the market
// calendar, which needs the real instant rather than `ctx.today`.
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
try checks.append(arena, try checkCandleFreshness(arena, &store, &tracked, now_s));
}
// Hand-maintained data staleness.
@ -1425,3 +1495,119 @@ test "trimTrailingSlash: drops a single trailing slash" {
try testing.expectEqualStrings("https://h", trimTrailingSlash("https://h/"));
try testing.expectEqualStrings("https://h", trimTrailingSlash("https://h"));
}
/// Seed a cache directory with `candles_meta.srf` for each symbol/date pair.
fn seedCandleMeta(io: std.Io, tmp: *std.testing.TmpDir, pairs: []const struct { []const u8, []const u8 }) !void {
for (pairs) |p| {
try tmp.dir.createDir(io, p[0], std.Io.File.Permissions.default_dir);
const rel = try std.fmt.allocPrint(testing.allocator, "{s}/candles_meta.srf", .{p[0]});
defer testing.allocator.free(rel);
const f = try tmp.dir.createFile(io, rel, .{});
defer f.close(io);
var buf: [256]u8 = undefined;
var w = f.writer(io, &buf);
try w.interface.print(
"#!srfv1\n#!expires=99999999999\nlast_close:num:100.00,last_date::{s},provider::tiingo\n",
.{p[1]},
);
try w.interface.flush();
}
}
test "checkCandleFreshness: warns and names the laggards, never fails" {
const allocator = testing.allocator;
const io = testing.io;
var arena_state = std.heap.ArenaAllocator.init(allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
try seedCandleMeta(io, &tmp, &.{
.{ "AAPL", "2025-06-13" },
.{ "MSFT", "2025-06-13" },
.{ "NVDA", "2025-06-13" },
.{ "TSLA", "2025-06-12" },
});
var tracked = std.StringHashMap(void).init(arena);
for ([_][]const u8{ "AAPL", "MSFT", "NVDA", "TSLA" }) |s| try tracked.put(s, {});
var store = cache.Store.init(io, arena, dir_path);
// Friday 2025-06-13, 22:00 UTC - past the equity boundary.
const now_s = zfin.Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour;
const c = try checkCandleFreshness(arena, &store, &tracked, now_s);
// WARN, not FAIL: `doctor` exits non-zero on any fail, and a stale cache is
// a thing to fix rather than a broken install. Cron stays green.
try testing.expectEqual(Status.warn, c.status);
try testing.expect(std.mem.indexOf(u8, c.detail, "TSLA (1d)") != null);
try testing.expect(std.mem.indexOf(u8, c.detail, "zfin cache stale") != null);
// The three current symbols are not named.
try testing.expect(std.mem.indexOf(u8, c.detail, "AAPL") == null);
}
test "checkCandleFreshness: a caught-up corpus is OK" {
const allocator = testing.allocator;
const io = testing.io;
var arena_state = std.heap.ArenaAllocator.init(allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
try seedCandleMeta(io, &tmp, &.{ .{ "AAPL", "2025-06-13" }, .{ "MSFT", "2025-06-13" } });
var tracked = std.StringHashMap(void).init(arena);
for ([_][]const u8{ "AAPL", "MSFT" }) |s| try tracked.put(s, {});
var store = cache.Store.init(io, arena, dir_path);
const now_s = zfin.Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour;
const c = try checkCandleFreshness(arena, &store, &tracked, now_s);
try testing.expectEqual(Status.ok, c.status);
}
test "checkCandleFreshness: an empty cache is informational, not a warning" {
const allocator = testing.allocator;
const io = testing.io;
var arena_state = std.heap.ArenaAllocator.init(allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
// First run on a fresh machine must not look like a problem.
var tracked = std.StringHashMap(void).init(arena);
var store = cache.Store.init(io, arena, "/nonexistent/zfin-cache");
const c = try checkCandleFreshness(arena, &store, &tracked, 0);
try testing.expectEqual(Status.info, c.status);
}
test "checkCandleFreshness: an untracked laggard does not warn" {
const allocator = testing.allocator;
const io = testing.io;
var arena_state = std.heap.ArenaAllocator.init(allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
try seedCandleMeta(io, &tmp, &.{
.{ "AAPL", "2025-06-13" },
.{ "MSFT", "2025-06-13" },
.{ "OLDCO", "2025-01-02" },
});
// OLDCO is cached but tracked by nothing - an orphan. Nothing refreshes it,
// so warning about it every run would train the operator to skip the line.
var tracked = std.StringHashMap(void).init(arena);
for ([_][]const u8{ "AAPL", "MSFT" }) |s| try tracked.put(s, {});
var store = cache.Store.init(io, arena, dir_path);
const now_s = zfin.Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour;
const c = try checkCandleFreshness(arena, &store, &tracked, now_s);
try testing.expectEqual(Status.ok, c.status);
}

140
src/service.zig
View file

@ -3141,6 +3141,39 @@ pub const DataService = struct {
// Write to local cache
var s = self.store();
// Never let the shared cache move a symbol BACKWARDS. The server's
// bytes are written verbatim, `#!expires=` included, so its view of
// freshness becomes the client's - and if the server's copy carries an
// older bar than the one already here, an unconditional write replaces
// good local data with worse and stamps it authoritative.
//
// Observed: the server's cron fetched at 17:00 ET, some symbols got that
// session's bar and some did not, and every one of them was stamped
// fresh until the next boundary. A client that had already fetched the
// newer bar would have had it overwritten and then believed the older
// one for a full day.
//
// Only candle data can be ordered this way, so only candle data is
// guarded; everything else falls through unchanged.
if (isCandleType(data_type)) {
if (serverBarRegression(&s, symbol, response.body)) |reg| {
// WARN, not debug. A shared cache should never be behind a
// client that draws from it: it is the tier with the refresh
// cron and the provider budget. When it happens, something on
// the server side has stopped keeping up, and every other client
// is being handed the same stale bar - so this is an operator
// event, not a diagnostic detail. Logged at debug, it was
// invisible in exactly the builds people install.
log.warn(
"{s}: shared cache is BEHIND this client for {s} - it offered {f}, local copy has {f}. Refused the sync rather than move the cache backwards; the server needs a refresh (see `zfin cache stale`).",
.{ symbol, @tagName(data_type), reg.incoming, reg.local },
);
log.debug("{s}: tryOneSync finished ({s}) result=ok elapsed_ms={d}", .{ symbol, @tagName(data_type), @divTrunc(std.Io.Timestamp.now(self.io, .awake).nanoseconds - t_start, std.time.ns_per_ms) });
return .ok;
}
}
s.writeRaw(symbol, data_type, response.body) catch |err| {
log.debug("{s}: failed to write synced {s} to cache: {s}", .{ symbol, @tagName(data_type), @errorName(err) });
log.debug("{s}: tryOneSync finished ({s}) result=net_err elapsed_ms={d}", .{ symbol, @tagName(data_type), @divTrunc(std.Io.Timestamp.now(self.io, .awake).nanoseconds - t_start, std.time.ns_per_ms) });
@ -3152,6 +3185,54 @@ pub const DataService = struct {
}
/// Sync candle data (both daily and meta) from the server.
/// Do these bytes describe candle data, i.e. data with a newest-bar date
/// that can be compared for age?
fn isCandleType(data_type: cache.DataType) bool {
return data_type == .candles_daily or data_type == .candles_meta;
}
/// Both dates, when writing `body` would replace the local candle data with
/// an OLDER bar. Null otherwise.
///
/// Returns the pair rather than a bool because the caller has to report
/// them: "the shared cache is behind you" is only actionable with the two
/// dates attached.
///
/// Null whenever the question cannot be answered - no local copy, an
/// unparseable body, no dates on either side - so an unknown never blocks a
/// sync. The guard only fires on a definite regression.
fn serverBarRegression(
s: *cache.Store,
symbol: []const u8,
body: []const u8,
) ?struct { local: Date, incoming: Date } {
const local = s.readCandleMeta(symbol) orelse return null;
const incoming = newestDateIn(body) orelse return null;
if (!incoming.lessThan(local.meta.last_date)) return null;
return .{ .local = local.meta.last_date, .incoming = incoming };
}
/// Newest `last_date::` or `date::` value in an SRF body.
///
/// Deliberately a scan for the maximum rather than a parse: `candles_meta`
/// carries one `last_date`, `candles_daily` carries a `date` per bar, and
/// the ordering of the latter is not something this check should assume.
fn newestDateIn(body: []const u8) ?Date {
var best: ?Date = null;
for ([_][]const u8{ "last_date::", "date::" }) |key| {
var rest = body;
while (std.mem.indexOf(u8, rest, key)) |idx| {
const start = idx + key.len;
rest = rest[start..];
const end = std.mem.indexOfAny(u8, rest, ",\n\r") orelse rest.len;
if (Date.parse(rest[0..end])) |d| {
if (best == null or best.?.lessThan(d)) best = d;
} else |_| {}
}
}
return best;
}
fn syncCandlesFromServer(self: *DataService, symbol: []const u8) bool {
const daily = self.syncFromServer(symbol, .candles_daily);
const meta = self.syncFromServer(symbol, .candles_meta);
@ -4907,3 +4988,62 @@ test "earningsNeedsRefresh: chase window is inclusive at the boundary" {
const past_window = [_]EarningsEvent{.{ .date = Date.fromYmd(2026, 6, 11), .estimate = 1.0 }};
try std.testing.expect(!DataService.earningsNeedsRefresh(&past_window, today, 14));
}
test "newestDateIn: picks the maximum across candles_meta and candles_daily shapes" {
// `candles_meta` carries one `last_date`; `candles_daily` carries a `date`
// per bar, in an order this check must not assume.
const meta = "#!srfv1\n#!expires=1\nlast_close:num:100.00,last_date::2026-08-07,provider::tiingo\n";
try std.testing.expect(DataService.newestDateIn(meta).?.eql(Date.fromYmd(2026, 8, 7)));
// Deliberately out of order.
const daily = "#!srfv1\ndate::2026-08-05,close:num:1\ndate::2026-08-07,close:num:3\ndate::2026-08-06,close:num:2\n";
try std.testing.expect(DataService.newestDateIn(daily).?.eql(Date.fromYmd(2026, 8, 7)));
// Nothing parseable -> null, so the guard cannot fire on garbage.
try std.testing.expectEqual(@as(?Date, null), DataService.newestDateIn("#!srfv1\n"));
try std.testing.expectEqual(@as(?Date, null), DataService.newestDateIn("last_date::not-a-date\n"));
}
test "isCandleType: only candle data is age-comparable" {
try std.testing.expect(DataService.isCandleType(.candles_daily));
try std.testing.expect(DataService.isCandleType(.candles_meta));
// Dividends, splits, options and the rest have no single newest-bar date,
// so they sync unguarded.
try std.testing.expect(!DataService.isCandleType(.dividends));
try std.testing.expect(!DataService.isCandleType(.splits));
try std.testing.expect(!DataService.isCandleType(.classification));
}
test "serverBarRegression: blocks a regression and reports both dates" {
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
var s = cache.Store.init(io, allocator, dir_path);
// Local copy holds Monday's bar.
s.updateCandleMeta("AAPL", 100.0, Date.fromYmd(2026, 8, 10), .tiingo, 0, 9999999999);
const older = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-07,provider::tiingo\n";
const same = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-10,provider::tiingo\n";
const newer = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-11,provider::tiingo\n";
// THE REGRESSION THIS GUARDS. The shared cache's cron fetched at 17:00 ET;
// some symbols got that session's bar and some did not, and all were
// stamped fresh until the next boundary. Written unconditionally, the older
// body replaces good local data and is then believed for a full day.
const reg = DataService.serverBarRegression(&s, "AAPL", older) orelse
return error.ExpectedRegression;
// Both dates come back, because the warning is only actionable with them.
try std.testing.expect(reg.local.eql(Date.fromYmd(2026, 8, 10)));
try std.testing.expect(reg.incoming.eql(Date.fromYmd(2026, 8, 7)));
try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", same));
try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", newer));
// Unknowns never block: no local copy, and an unparseable body.
try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "NOLOCAL", older));
try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", "#!srfv1\n"));
}