add a zfin cache stale command to find out of date candle data
This commit is contained in:
parent
2da30b2474
commit
79326cf41b
GPG key ID:
A7B62D657EF764F8
4 changed files with 673 additions and 25 deletions
434
src/cache/freshness.zig
vendored
Normal file
434
src/cache/freshness.zig
vendored
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
//! Corpus-wide candle-cache staleness detection.
|
||||
//!
|
||||
//! `market.candleFreshness` answers "is THIS symbol's newest bar overdue?" from
|
||||
//! the clock alone. That is the right question for a fetch gate and the wrong
|
||||
//! one for finding a symbol the cache has quietly frozen, because the clock
|
||||
//! cannot distinguish three cases that look identical per-symbol: the bar isn't
|
||||
//! published yet, the market was closed, or this symbol alone got left behind.
|
||||
//!
|
||||
//! Peers settle it. Symbols of the same `InstrumentKind` share a publication
|
||||
//! schedule, so if twenty equities hold Friday's bar and three hold Thursday's,
|
||||
//! those three are behind - no calendar reasoning required, and an un-modeled
|
||||
//! closure cannot fool it because a closure moves every symbol together, taking
|
||||
//! the peer maximum with them and leaving nothing behind it.
|
||||
//!
|
||||
//! That self-correction is why there is no calendar clamp on top. A first cut
|
||||
//! suppressed findings whenever the group's own newest bar was itself overdue,
|
||||
//! reasoning that a wholly-behind group meant a closure. It meant nothing of
|
||||
//! the kind: it meant the corpus had not been refreshed yet that day, which is
|
||||
//! the normal state, and the gate hid three genuinely-lagging symbols in live
|
||||
//! data. "Is the corpus caught up?" and "is any symbol behind the rest?" are
|
||||
//! independent questions, and only the second one names a symbol.
|
||||
//!
|
||||
//! The observed failure this exists for: three symbols served Thursday's close
|
||||
//! all weekend while ~19 peers had Friday's. Each had been fetched, missed,
|
||||
//! stamped `.lagging`, retried 30 minutes later, and by then sat just past the
|
||||
//! 90-minute provider-lag grace - so `candleFreshness` said `.overdue`,
|
||||
//! `staleCandleExpiry` routed them to the next session boundary, and the stale
|
||||
//! bars became authoritative until Monday. The retry machinery worked as
|
||||
//! designed; the premise was wrong.
|
||||
//!
|
||||
//! Pure: dates and `now_s` in, findings out. No I/O, no cache reads, no
|
||||
//! fetches. The caller supplies the corpus.
|
||||
|
||||
const std = @import("std");
|
||||
const Date = @import("../Date.zig");
|
||||
const market = @import("../market.zig");
|
||||
|
||||
/// One cached symbol, as the caller found it.
|
||||
pub const Entry = struct {
|
||||
symbol: []const u8,
|
||||
kind: market.InstrumentKind,
|
||||
/// Newest bar in the cache, or null when there is no candle meta at all -
|
||||
/// never fetched, negative-cached, or a key that carries no candles (an
|
||||
/// EDGAR CIK, say). Null entries are never reported stale: absence of
|
||||
/// evidence is not a stale bar.
|
||||
last_date: ?Date,
|
||||
/// Is this symbol in the set zfin intends to keep fresh
|
||||
/// (`Portfolio.fetchedSymbols`)? An untracked symbol can only ever fall
|
||||
/// further behind, so reporting it as stale every run would be noise.
|
||||
tracked: bool,
|
||||
};
|
||||
|
||||
/// A tracked symbol whose newest bar is behind its peers'.
|
||||
pub const Finding = struct {
|
||||
symbol: []const u8,
|
||||
kind: market.InstrumentKind,
|
||||
last_date: Date,
|
||||
/// Newest bar held by any same-kind peer.
|
||||
peer_date: Date,
|
||||
/// Calendar days between the two. Calendar rather than trading days
|
||||
/// because this is a human-facing magnitude, and the distinction between
|
||||
/// "one session" and "six weeks" is what it has to convey.
|
||||
days_behind: i64,
|
||||
|
||||
fn worstFirst(_: void, a: Finding, b: Finding) bool {
|
||||
if (a.days_behind != b.days_behind) return a.days_behind > b.days_behind;
|
||||
return std.mem.order(u8, a.symbol, b.symbol) == .lt;
|
||||
}
|
||||
};
|
||||
|
||||
/// Per-`InstrumentKind` reference state, so a caller can explain itself rather
|
||||
/// than just listing symbols.
|
||||
pub const GroupState = struct {
|
||||
kind: market.InstrumentKind,
|
||||
/// Newest bar across every dated entry of this kind; null when the kind
|
||||
/// has no dated entries.
|
||||
peer_date: ?Date,
|
||||
/// Freshness of the group's OWN newest bar - context, NOT a gate.
|
||||
///
|
||||
/// Reported so a caller can say "the whole corpus is a session behind,
|
||||
/// refresh everything" alongside any per-symbol findings. It deliberately
|
||||
/// does not suppress those findings: an earlier version gated on
|
||||
/// `.current` and thereby hid three genuinely-lagging symbols every time
|
||||
/// the corpus as a whole had not been refreshed that day, which is most of
|
||||
/// the time.
|
||||
///
|
||||
/// No clamp is needed, because peer comparison is already immune to the
|
||||
/// case a clamp was meant to catch: a market closure moves every symbol
|
||||
/// together, so the peer maximum moves with them and nothing sits behind
|
||||
/// it. The two questions are independent - "is the corpus caught up?" and
|
||||
/// "is any symbol behind the rest?" - and only the second one identifies a
|
||||
/// symbol.
|
||||
freshness: ?market.CandleFreshness,
|
||||
/// Dated entries considered for this kind.
|
||||
dated: usize,
|
||||
|
||||
/// Can this kind yield findings at all? Only that a peer exists to compare
|
||||
/// against; with a single cached symbol there is no comparison to make.
|
||||
pub fn conclusive(self: GroupState) bool {
|
||||
return self.dated > 1;
|
||||
}
|
||||
};
|
||||
|
||||
pub const Report = struct {
|
||||
/// Tracked symbols behind their peers, worst first.
|
||||
stale: []Finding,
|
||||
/// Cached, dated, but tracked by nothing. Nothing will ever refresh these,
|
||||
/// so they are not staleness - they are disk you may want back. Reported
|
||||
/// separately precisely so they cannot dominate the stale list forever.
|
||||
orphans: []const []const u8,
|
||||
/// Tracked but with no cached bar at all. A different finding: not stale,
|
||||
/// never fetched.
|
||||
missing: []const []const u8,
|
||||
groups: []GroupState,
|
||||
|
||||
pub fn deinit(self: Report, allocator: std.mem.Allocator) void {
|
||||
allocator.free(self.stale);
|
||||
allocator.free(self.orphans);
|
||||
allocator.free(self.missing);
|
||||
allocator.free(self.groups);
|
||||
}
|
||||
};
|
||||
|
||||
/// Classify `entries` as of `now_s`.
|
||||
///
|
||||
/// Symbol strings are borrowed from `entries`; only the slices are allocated.
|
||||
pub fn scan(
|
||||
allocator: std.mem.Allocator,
|
||||
entries: []const Entry,
|
||||
now_s: i64,
|
||||
) !Report {
|
||||
const kinds = [_]market.InstrumentKind{ .equity, .mutual_fund };
|
||||
|
||||
var groups = try allocator.alloc(GroupState, kinds.len);
|
||||
errdefer allocator.free(groups);
|
||||
for (kinds, 0..) |kind, i| {
|
||||
var newest: ?Date = null;
|
||||
var dated: usize = 0;
|
||||
for (entries) |e| {
|
||||
if (e.kind != kind) continue;
|
||||
const d = e.last_date orelse continue;
|
||||
dated += 1;
|
||||
if (newest == null or newest.?.lessThan(d)) newest = d;
|
||||
}
|
||||
groups[i] = .{
|
||||
.kind = kind,
|
||||
.peer_date = newest,
|
||||
// Context only (see GroupState.freshness). Reuses the fetch
|
||||
// gate's own verdict rather than re-deriving the market calendar.
|
||||
.freshness = if (newest) |n| market.candleFreshness(now_s, kind, n) else null,
|
||||
.dated = dated,
|
||||
};
|
||||
}
|
||||
|
||||
var stale = std.ArrayList(Finding).empty;
|
||||
errdefer stale.deinit(allocator);
|
||||
var orphans = std.ArrayList([]const u8).empty;
|
||||
errdefer orphans.deinit(allocator);
|
||||
var missing = std.ArrayList([]const u8).empty;
|
||||
errdefer missing.deinit(allocator);
|
||||
|
||||
for (entries) |e| {
|
||||
const last = e.last_date orelse {
|
||||
// No bar at all. Only worth saying when something tracks it.
|
||||
if (e.tracked) try missing.append(allocator, e.symbol);
|
||||
continue;
|
||||
};
|
||||
if (!e.tracked) {
|
||||
try orphans.append(allocator, e.symbol);
|
||||
continue;
|
||||
}
|
||||
|
||||
const g = groups[if (e.kind == .equity) 0 else 1];
|
||||
if (!g.conclusive()) continue;
|
||||
const peer = g.peer_date.?;
|
||||
if (!last.lessThan(peer)) continue;
|
||||
|
||||
try stale.append(allocator, .{
|
||||
.symbol = e.symbol,
|
||||
.kind = e.kind,
|
||||
.last_date = last,
|
||||
.peer_date = peer,
|
||||
.days_behind = @divTrunc(peer.toEpoch() - last.toEpoch(), std.time.s_per_day),
|
||||
});
|
||||
}
|
||||
|
||||
// Ownership moves out of the ArrayLists one at a time, so each completed
|
||||
// handoff needs its own errdefer: the list's errdefer no longer covers a
|
||||
// slice it has released, and the Report does not exist yet to cover it
|
||||
// either. An allocation-failure test caught `stale_out` leaking in exactly
|
||||
// that window.
|
||||
const stale_out = try stale.toOwnedSlice(allocator);
|
||||
errdefer allocator.free(stale_out);
|
||||
std.mem.sort(Finding, stale_out, {}, Finding.worstFirst);
|
||||
|
||||
const orphans_out = try orphans.toOwnedSlice(allocator);
|
||||
errdefer allocator.free(orphans_out);
|
||||
|
||||
const missing_out = try missing.toOwnedSlice(allocator);
|
||||
|
||||
return .{
|
||||
.stale = stale_out,
|
||||
.orphans = orphans_out,
|
||||
.missing = missing_out,
|
||||
.groups = groups,
|
||||
};
|
||||
}
|
||||
|
||||
// ── tests ────────────────────────────────────────────────────
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
/// Friday 2025-06-13 at 18:00 ET, past the 16:55 equity boundary. Equities
|
||||
/// should hold Friday's bar; mutual-fund NAVs for Friday are not published
|
||||
/// until the next morning, so funds should hold Thursday's.
|
||||
fn fridayEvening() i64 {
|
||||
return Date.fromYmd(2025, 6, 13).toEpoch() + 22 * std.time.s_per_hour;
|
||||
}
|
||||
|
||||
fn eq(sym: []const u8, d: ?Date, tracked: bool) Entry {
|
||||
return .{ .symbol = sym, .kind = .equity, .last_date = d, .tracked = tracked };
|
||||
}
|
||||
|
||||
test "scan: a symbol behind its peers is reported" {
|
||||
const a = testing.allocator;
|
||||
// The shape of the observed failure: most of the corpus has the latest
|
||||
// bar, a few do not.
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const thu = Date.fromYmd(2025, 6, 12);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
eq("NVDA", fri, true),
|
||||
eq("TSLA", thu, true),
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
|
||||
try testing.expectEqual(@as(usize, 1), r.stale.len);
|
||||
try testing.expectEqualStrings("TSLA", r.stale[0].symbol);
|
||||
try testing.expectEqual(@as(i64, 1), r.stale[0].days_behind);
|
||||
try testing.expect(r.stale[0].peer_date.eql(fri));
|
||||
}
|
||||
|
||||
test "scan: a uniformly-behind group reports nothing, and needs no clamp to do it" {
|
||||
const a = testing.allocator;
|
||||
// A closure (or simply a corpus nobody refreshed) moves every symbol
|
||||
// together, so the peer maximum moves too and no symbol sits behind it.
|
||||
// Peer comparison is self-correcting here - a clock-based check would flag
|
||||
// all four, and an added calendar clamp would suppress real findings
|
||||
// elsewhere for no gain. `freshness` is still reported as context so a
|
||||
// caller can say "refresh everything".
|
||||
const wed = Date.fromYmd(2025, 6, 11);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", wed, true),
|
||||
eq("MSFT", wed, true),
|
||||
eq("NVDA", wed, true),
|
||||
eq("TSLA", wed, true),
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), r.stale.len);
|
||||
// Context says the corpus is behind; it does not gate anything.
|
||||
try testing.expect(r.groups[0].freshness.? != .current);
|
||||
try testing.expect(r.groups[0].conclusive());
|
||||
}
|
||||
|
||||
test "scan: a laggard is found even when the corpus as a whole is behind" {
|
||||
const a = testing.allocator;
|
||||
// THE REGRESSION THIS GUARDS, caught on live data. Monday evening, nothing
|
||||
// has refreshed since Friday: every symbol is behind the session the
|
||||
// calendar expects. An earlier version gated findings on the group's own
|
||||
// bar being `.current` and so reported nothing at all - while three
|
||||
// symbols sat a full session behind the other twenty-one.
|
||||
//
|
||||
// Monday 2025-06-16 22:00 UTC, past the equity boundary, so Monday's bar
|
||||
// is expected and even the peer maximum (Friday) is overdue.
|
||||
const monday_evening = Date.fromYmd(2025, 6, 16).toEpoch() + 22 * std.time.s_per_hour;
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const thu = Date.fromYmd(2025, 6, 12);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
eq("NVDA", fri, true),
|
||||
eq("AGG", thu, true),
|
||||
eq("AMZN", thu, true),
|
||||
eq("NKE", thu, true),
|
||||
};
|
||||
var r = try scan(a, &entries, monday_evening);
|
||||
defer r.deinit(a);
|
||||
|
||||
// The group is behind - and that must not silence the three outliers.
|
||||
try testing.expect(r.groups[0].freshness.? != .current);
|
||||
try testing.expectEqual(@as(usize, 3), r.stale.len);
|
||||
for (r.stale) |f| {
|
||||
try testing.expect(f.last_date.eql(thu));
|
||||
try testing.expect(f.peer_date.eql(fri));
|
||||
try testing.expectEqual(@as(i64, 1), f.days_behind);
|
||||
}
|
||||
}
|
||||
|
||||
test "scan: kinds are judged separately, so fund NAV lag is not staleness" {
|
||||
const a = testing.allocator;
|
||||
// Friday evening: equities have Friday, funds still legitimately have
|
||||
// Thursday. Judged as one corpus, every fund would read a day stale every
|
||||
// single evening.
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const thu = Date.fromYmd(2025, 6, 12);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
.{ .symbol = "VBTLX", .kind = .mutual_fund, .last_date = thu, .tracked = true },
|
||||
.{ .symbol = "VPMAX", .kind = .mutual_fund, .last_date = thu, .tracked = true },
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), r.stale.len);
|
||||
// Both groups are caught up against their own schedules.
|
||||
try testing.expectEqual(market.CandleFreshness.current, r.groups[0].freshness.?);
|
||||
try testing.expectEqual(market.CandleFreshness.current, r.groups[1].freshness.?);
|
||||
}
|
||||
|
||||
test "scan: a lone symbol in its kind yields no verdict" {
|
||||
const a = testing.allocator;
|
||||
// With one peer there is nothing to compare against, and the clock alone
|
||||
// cannot tell a frozen cache from a closed market. Silence is the honest
|
||||
// answer, and this is peer comparison's documented blind spot.
|
||||
const entries = [_]Entry{
|
||||
.{ .symbol = "VBTLX", .kind = .mutual_fund, .last_date = Date.fromYmd(2025, 5, 1), .tracked = true },
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
try testing.expectEqual(@as(usize, 0), r.stale.len);
|
||||
try testing.expect(!r.groups[1].conclusive());
|
||||
try testing.expectEqual(@as(usize, 1), r.groups[1].dated);
|
||||
}
|
||||
|
||||
test "scan: an untracked symbol is an orphan, never stale" {
|
||||
const a = testing.allocator;
|
||||
// Nothing refreshes it, so it can only fall further behind. Reporting it
|
||||
// as stale would put a permanent entry at the top of the list and train
|
||||
// the operator to skip the section.
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
eq("OLDCO", Date.fromYmd(2025, 1, 2), false),
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), r.stale.len);
|
||||
try testing.expectEqual(@as(usize, 1), r.orphans.len);
|
||||
try testing.expectEqualStrings("OLDCO", r.orphans[0]);
|
||||
}
|
||||
|
||||
test "scan: no candle meta means missing when tracked, ignored when not" {
|
||||
const a = testing.allocator;
|
||||
// A negative-cached symbol or a non-candle key has no bar. That is not a
|
||||
// stale bar, and inventing an infinitely-old date for it would put it
|
||||
// permanently at the top of the worst-first list.
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
eq("DOGE-USD", null, true),
|
||||
eq("0000320193", null, false),
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), r.stale.len);
|
||||
try testing.expectEqual(@as(usize, 0), r.orphans.len);
|
||||
try testing.expectEqual(@as(usize, 1), r.missing.len);
|
||||
try testing.expectEqualStrings("DOGE-USD", r.missing[0]);
|
||||
try testing.expectEqual(@as(usize, 2), r.groups[0].dated);
|
||||
}
|
||||
|
||||
test "scan: findings are ordered worst first" {
|
||||
const a = testing.allocator;
|
||||
// A symbol one session behind and one six weeks behind are different
|
||||
// diagnoses; the second should not be buried under the first.
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
eq("NEAR", Date.fromYmd(2025, 6, 12), true),
|
||||
eq("FAR", Date.fromYmd(2025, 5, 2), true),
|
||||
eq("MID", Date.fromYmd(2025, 6, 6), true),
|
||||
};
|
||||
var r = try scan(a, &entries, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
|
||||
try testing.expectEqual(@as(usize, 3), r.stale.len);
|
||||
try testing.expectEqualStrings("FAR", r.stale[0].symbol);
|
||||
try testing.expectEqualStrings("MID", r.stale[1].symbol);
|
||||
try testing.expectEqualStrings("NEAR", r.stale[2].symbol);
|
||||
try testing.expectEqual(@as(i64, 42), r.stale[0].days_behind);
|
||||
}
|
||||
|
||||
test "scan: an empty corpus is not an error" {
|
||||
const a = testing.allocator;
|
||||
var r = try scan(a, &.{}, fridayEvening());
|
||||
defer r.deinit(a);
|
||||
try testing.expectEqual(@as(usize, 0), r.stale.len);
|
||||
for (r.groups) |g| {
|
||||
try testing.expectEqual(@as(?Date, null), g.peer_date);
|
||||
try testing.expect(!g.conclusive());
|
||||
}
|
||||
}
|
||||
|
||||
/// OOM-path wrapper for `checkAllAllocationFailures`.
|
||||
fn scanOom(a: std.mem.Allocator, entries: []const Entry, now_s: i64) !void {
|
||||
var r = try scan(a, entries, now_s);
|
||||
r.deinit(a);
|
||||
}
|
||||
|
||||
test "scan: every allocation-failure path unwinds cleanly" {
|
||||
const fri = Date.fromYmd(2025, 6, 13);
|
||||
const entries = [_]Entry{
|
||||
eq("AAPL", fri, true),
|
||||
eq("MSFT", fri, true),
|
||||
eq("TSLA", Date.fromYmd(2025, 6, 12), true),
|
||||
eq("OLDCO", Date.fromYmd(2025, 1, 2), false),
|
||||
eq("DOGE-USD", null, true),
|
||||
};
|
||||
try testing.checkAllAllocationFailures(
|
||||
testing.allocator,
|
||||
scanOom,
|
||||
.{ @as([]const Entry, &entries), fridayEvening() },
|
||||
);
|
||||
}
|
||||
|
|
@ -3,11 +3,13 @@ const zfin = @import("../root.zig");
|
|||
const cli = @import("common.zig");
|
||||
const framework = @import("framework.zig");
|
||||
const srf = @import("srf");
|
||||
const freshness = @import("../cache/freshness.zig");
|
||||
const projections = @import("../analytics/projections.zig");
|
||||
|
||||
const Store = zfin.cache.Store;
|
||||
const DataType = zfin.cache.DataType;
|
||||
|
||||
pub const Subcommand = enum { stats, clear };
|
||||
pub const Subcommand = enum { stats, stale, clear };
|
||||
|
||||
pub const ParsedArgs = struct {
|
||||
sub: Subcommand,
|
||||
|
|
@ -18,13 +20,19 @@ pub const meta: framework.Meta = .{
|
|||
.group = .infra,
|
||||
.synopsis = "Inspect or clear the local provider-data cache",
|
||||
.help =
|
||||
\\Usage: zfin cache <stats|clear>
|
||||
\\Usage: zfin cache <stats|stale|clear>
|
||||
\\
|
||||
\\Subcommands:
|
||||
\\ stats List every cached symbol with per-data-type size,
|
||||
\\ age, and freshness state. Stale entries (past TTL)
|
||||
\\ are flagged. Includes the cusip_tickers.srf file
|
||||
\\ if present.
|
||||
\\ stale Find symbols whose newest candle is behind their
|
||||
\\ peers'. Compares each symbol against others of the
|
||||
\\ same kind (equity vs mutual fund) rather than against
|
||||
\\ the clock, so an un-modeled market closure - which
|
||||
\\ moves every symbol together - is not mistaken for a
|
||||
\\ frozen cache. Read-only; reports, never fetches.
|
||||
\\ clear Delete every file under the cache directory.
|
||||
\\ No confirmation; the next provider call will
|
||||
\\ re-fetch everything.
|
||||
|
|
@ -55,7 +63,7 @@ const display_labels = [_][]const u8{
|
|||
|
||||
pub fn parseArgs(ctx: *framework.RunCtx, cmd_args: []const []const u8) !ParsedArgs {
|
||||
if (cmd_args.len < 1) {
|
||||
cli.stderrPrint(ctx.io, "Error: 'cache' requires a subcommand (stats, clear)\n");
|
||||
cli.stderrPrint(ctx.io, "Error: 'cache' requires a subcommand (stats, stale, clear)\n");
|
||||
return error.MissingSubcommand;
|
||||
}
|
||||
if (cmd_args.len > 1) {
|
||||
|
|
@ -66,18 +74,22 @@ pub fn parseArgs(ctx: *framework.RunCtx, cmd_args: []const []const u8) !ParsedAr
|
|||
if (std.mem.eql(u8, sub_str, "stats")) {
|
||||
return .{ .sub = .stats };
|
||||
}
|
||||
if (std.mem.eql(u8, sub_str, "stale")) {
|
||||
return .{ .sub = .stale };
|
||||
}
|
||||
if (std.mem.eql(u8, sub_str, "clear")) {
|
||||
return .{ .sub = .clear };
|
||||
}
|
||||
cli.stderrPrint(ctx.io, "Error: unknown cache subcommand '");
|
||||
cli.stderrPrint(ctx.io, sub_str);
|
||||
cli.stderrPrint(ctx.io, "'. Use 'stats' or 'clear'.\n");
|
||||
cli.stderrPrint(ctx.io, "'. Use 'stats', 'stale' or 'clear'.\n");
|
||||
return error.UnknownSubcommand;
|
||||
}
|
||||
|
||||
pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
|
||||
switch (parsed.sub) {
|
||||
.stats => try runStats(ctx),
|
||||
.stale => try runStale(ctx),
|
||||
.clear => try runClear(ctx),
|
||||
}
|
||||
}
|
||||
|
|
@ -191,6 +203,147 @@ fn runStats(ctx: *framework.RunCtx) !void {
|
|||
});
|
||||
}
|
||||
|
||||
/// Peer-comparison staleness sweep over the cached corpus.
|
||||
///
|
||||
/// Read-only by design: it reports and never fetches, so it is safe to run
|
||||
/// against a shared cache and safe to put in front of an operator who has not
|
||||
/// decided what to do yet. `zfin cache refresh` is the acting half.
|
||||
fn runStale(ctx: *framework.RunCtx) !void {
|
||||
const io = ctx.io;
|
||||
const allocator = ctx.allocator;
|
||||
const out = ctx.out;
|
||||
|
||||
// wall-clock required: the sweep compares each symbol's newest bar against
|
||||
// the market calendar's notion of what should be published by now.
|
||||
// Captured once so every symbol is judged against the same instant.
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
|
||||
// Arena for the transient string work below. Not a nicety: the tracked-set
|
||||
// keys are borrowed by a hashmap that outlives the block that builds them,
|
||||
// and a scoped `defer free` there dangles every key before the sweep reads
|
||||
// it. One arena at function scope removes that hazard and the two smaller
|
||||
// leaks around it.
|
||||
var arena_state = std.heap.ArenaAllocator.init(allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
var store = Store.init(io, allocator, ctx.config.cache_dir);
|
||||
const keys = store.cacheKeys(arena) catch {
|
||||
try out.print("Cache is empty; nothing to check.\n", .{});
|
||||
return;
|
||||
};
|
||||
if (keys.len == 0) {
|
||||
try out.print("Cache is empty; nothing to check.\n", .{});
|
||||
return;
|
||||
}
|
||||
|
||||
// The set zfin intends to keep fresh. Absent a portfolio every cached
|
||||
// symbol counts as untracked, which is honest - without one there is
|
||||
// nothing to be tracked BY.
|
||||
var tracked = std.StringHashMap(void).init(arena);
|
||||
if (cli.loadPortfolio(ctx, ctx.today)) |loaded_pf| {
|
||||
var l = loaded_pf;
|
||||
defer l.deinit(allocator);
|
||||
const wl = ctx.resolveWatchlistPath();
|
||||
defer wl.deinit(allocator);
|
||||
const wl_syms: ?[][]const u8 = if (ctx.globals.watchlist_path != null or wl.resolved != null)
|
||||
cli.loadWatchlist(io, arena, wl.path)
|
||||
else
|
||||
null;
|
||||
|
||||
const syms = try l.portfolio.fetchedSymbols(arena, .{
|
||||
.watchlist_syms = if (wl_syms) |w| w else &.{},
|
||||
.benchmarks = benchmarkPair(io, arena, l.anchor()),
|
||||
});
|
||||
for (syms) |sym| try tracked.put(sym, {});
|
||||
}
|
||||
|
||||
// Build the corpus. `readCandleMeta` returning null IS the filter for
|
||||
// non-candle keys: EDGAR CIKs, negative-cached symbols, and anything never
|
||||
// fetched all land here as `last_date = null`, and none of them can be
|
||||
// reported stale.
|
||||
var entries = std.ArrayList(freshness.Entry).empty;
|
||||
for (keys) |key| {
|
||||
const cm = store.readCandleMeta(key);
|
||||
try entries.append(arena, .{
|
||||
.symbol = key,
|
||||
.kind = zfin.market.classify(key),
|
||||
.last_date = if (cm) |m| m.meta.last_date else null,
|
||||
.tracked = tracked.contains(key),
|
||||
});
|
||||
}
|
||||
|
||||
var report = try freshness.scan(arena, entries.items, now_s);
|
||||
defer report.deinit(arena);
|
||||
|
||||
for (report.groups) |g| {
|
||||
const label = switch (g.kind) {
|
||||
.equity => "equity/ETF",
|
||||
.mutual_fund => "mutual fund",
|
||||
};
|
||||
if (g.peer_date) |pd| {
|
||||
try out.print("{s:<12} {d} cached, newest bar {f}", .{ label, g.dated, pd });
|
||||
if (!g.conclusive()) {
|
||||
try out.print(" - only one cached, no peer to compare against", .{});
|
||||
} else if (g.freshness) |f| switch (f) {
|
||||
// Advice about the corpus, NOT a reason to withhold findings.
|
||||
// The per-symbol comparison below stands on its own.
|
||||
.lagging => try out.print(" - the latest session's bar is not published yet", .{}),
|
||||
.overdue => try out.print(" - the whole group is a session or more behind; a refresh is due", .{}),
|
||||
.current => {},
|
||||
};
|
||||
try out.print("\n", .{});
|
||||
} else {
|
||||
try out.print("{s:<12} none cached\n", .{label});
|
||||
}
|
||||
}
|
||||
|
||||
if (report.stale.len > 0) {
|
||||
try out.print("\nBehind their peers ({d}):\n", .{report.stale.len});
|
||||
for (report.stale) |f| {
|
||||
try out.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date });
|
||||
}
|
||||
try out.print("\n zfin --refresh-data=force portfolio re-fetches the tracked set\n", .{});
|
||||
} else {
|
||||
try out.print("\nNothing behind its peers.\n", .{});
|
||||
}
|
||||
|
||||
if (report.missing.len > 0) {
|
||||
try out.print("\nTracked but never cached ({d}): ", .{report.missing.len});
|
||||
for (report.missing, 0..) |sym, i| try out.print("{s}{s}", .{ if (i == 0) "" else ", ", sym });
|
||||
try out.print("\n", .{});
|
||||
}
|
||||
|
||||
if (report.orphans.len > 0) {
|
||||
try out.print("\nCached but tracked by nothing ({d}): ", .{report.orphans.len});
|
||||
for (report.orphans, 0..) |sym, i| try out.print("{s}{s}", .{ if (i == 0) "" else ", ", sym });
|
||||
try out.print("\n Nothing will refresh these. `zfin cache clear` is the blunt option.\n", .{});
|
||||
}
|
||||
}
|
||||
|
||||
/// The projections benchmark pair, read from `projections.srf` beside the
|
||||
/// portfolio. Returns the SPY/AGG defaults when the file is absent or silent.
|
||||
///
|
||||
/// These belong in the tracked set even though they are held nowhere: they are
|
||||
/// fetched for the benchmark comparison, which is why AGG went stale unnoticed
|
||||
/// while SPY - which doubles as a `ticker::` alias on a real holding - stayed
|
||||
/// current.
|
||||
///
|
||||
/// `views/projections.zig` reads the same file for the same config, but keeps
|
||||
/// its `UserConfig` alive and reads the fields in place. This exists only
|
||||
/// because the tracked set outlives the config: an overridden symbol lives in a
|
||||
/// `[16]u8` field inside `UserConfig`, so the slices must be copied out before
|
||||
/// that struct dies.
|
||||
fn benchmarkPair(io: std.Io, arena: std.mem.Allocator, anchor: []const u8) []const []const u8 {
|
||||
const path = cli.siblingPath(arena, anchor, "projections.srf") catch return &.{};
|
||||
const data = std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024)) catch null;
|
||||
const cfg = projections.parseProjectionsConfig(data);
|
||||
const pair = arena.alloc([]const u8, 2) catch return &.{};
|
||||
pair[0] = arena.dupe(u8, cfg.benchmark_stock) catch return &.{};
|
||||
pair[1] = arena.dupe(u8, cfg.benchmark_bond) catch return &.{};
|
||||
return pair;
|
||||
}
|
||||
|
||||
fn runClear(ctx: *framework.RunCtx) !void {
|
||||
var store = Store.init(ctx.io, ctx.allocator, ctx.config.cache_dir);
|
||||
try store.clearAll();
|
||||
|
|
@ -392,3 +545,54 @@ test "formatSize: megabytes" {
|
|||
try std.testing.expectEqualStrings("1.0 MB", formatSize(&buf, 1024 * 1024));
|
||||
try std.testing.expectEqualStrings("2.5 MB", formatSize(&buf, 2 * 1024 * 1024 + 512 * 1024));
|
||||
}
|
||||
|
||||
test "parseArgs: 'stale' resolves to .stale" {
|
||||
var ctx: framework.RunCtx = undefined;
|
||||
ctx.io = std.testing.io;
|
||||
const args = [_][]const u8{"stale"};
|
||||
const parsed = try parseArgs(&ctx, &args);
|
||||
try std.testing.expectEqual(Subcommand.stale, parsed.sub);
|
||||
}
|
||||
|
||||
test "benchmarkPair: defaults to SPY/AGG when projections.srf is absent" {
|
||||
// These two must be in the tracked set even though they are held nowhere.
|
||||
// AGG went stale unnoticed for exactly this reason - it is fetched only for
|
||||
// the benchmark comparison, while SPY looked fine because it doubles as a
|
||||
// `ticker::` alias on a real holding.
|
||||
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const pair = benchmarkPair(std.testing.io, arena_state.allocator(), "/nonexistent/portfolio.srf");
|
||||
try std.testing.expectEqual(@as(usize, 2), pair.len);
|
||||
try std.testing.expectEqualStrings("SPY", pair[0]);
|
||||
try std.testing.expectEqualStrings("AGG", pair[1]);
|
||||
}
|
||||
|
||||
test "benchmarkPair: an override in projections.srf is honoured" {
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
{
|
||||
const f = try tmp.dir.createFile(io, "projections.srf", .{});
|
||||
defer f.close(io);
|
||||
var buf: [256]u8 = undefined;
|
||||
var w = f.writer(io, &buf);
|
||||
try w.interface.writeAll("#!srfv1\ntype::config,benchmark_stock::VTI,benchmark_bond::BND\n");
|
||||
try w.interface.flush();
|
||||
}
|
||||
|
||||
var arena_state = std.heap.ArenaAllocator.init(allocator);
|
||||
defer arena_state.deinit();
|
||||
const anchor = try std.fs.path.join(allocator, &.{ dir_path, "portfolio.srf" });
|
||||
defer allocator.free(anchor);
|
||||
|
||||
// Duped out of the config's stack-local override buffers - borrowing them
|
||||
// would dangle the moment benchmarkPair returned.
|
||||
const pair = benchmarkPair(io, arena_state.allocator(), anchor);
|
||||
try std.testing.expectEqual(@as(usize, 2), pair.len);
|
||||
try std.testing.expectEqualStrings("VTI", pair[0]);
|
||||
try std.testing.expectEqualStrings("BND", pair[1]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -389,6 +389,21 @@ fn printLoadSummaryImpl(io: std.Io, color: bool, s: LoadSummaryStats) !void {
|
|||
|
||||
const portfolio_loader = @import("../portfolio_loader.zig");
|
||||
|
||||
/// A path to `name` in the same directory as `anchor`.
|
||||
///
|
||||
/// The portfolio's siblings - `accounts.srf`, `metadata.srf`,
|
||||
/// `transaction_log.srf`, `projections.srf` - all live beside whichever
|
||||
/// portfolio file was resolved, so "next to the anchor" is the one rule that
|
||||
/// finds them regardless of `ZFIN_HOME`, `-p` patterns, or cwd.
|
||||
///
|
||||
/// Shared rather than per-command: it was private to `doctor` and the second
|
||||
/// caller promptly hand-rolled dirname + join, which is how the two would have
|
||||
/// drifted on trailing-separator handling.
|
||||
pub fn siblingPath(arena: std.mem.Allocator, anchor: []const u8, name: []const u8) ![]const u8 {
|
||||
const dir_end = if (std.mem.lastIndexOfScalar(u8, anchor, std.fs.path.sep)) |idx| idx + 1 else 0;
|
||||
return std.fmt.allocPrint(arena, "{s}{s}", .{ anchor[0..dir_end], name });
|
||||
}
|
||||
|
||||
pub const LoadedPortfolio = portfolio_loader.LoadedPortfolio;
|
||||
pub const PortfolioData = portfolio_loader.PortfolioData;
|
||||
pub const loadPortfolioFromConfig = portfolio_loader.loadPortfolioFromConfig;
|
||||
|
|
@ -1475,3 +1490,12 @@ test "buildPortfolioData: builds summary + candle_map for stock positions" {
|
|||
try std.testing.expect(pf_data.summary.allocations.len > 0);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 20_000), pf_data.summary.total_value, 1.0);
|
||||
}
|
||||
|
||||
test "siblingPath: joins a filename onto the anchor's directory" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const a = arena.allocator();
|
||||
try std.testing.expectEqualStrings("/home/u/data/accounts.srf", try siblingPath(a, "/home/u/data/portfolio.srf", "accounts.srf"));
|
||||
// Bare filename (no separator) -> sibling is just the name.
|
||||
try std.testing.expectEqualStrings("accounts.srf", try siblingPath(a, "portfolio.srf", "accounts.srf"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -457,11 +457,6 @@ fn checkSrfFile(
|
|||
}
|
||||
|
||||
/// Join a sibling filename onto the anchor portfolio's directory.
|
||||
fn siblingPath(arena: std.mem.Allocator, anchor: []const u8, name: []const u8) ![]const u8 {
|
||||
const dir_end = if (std.mem.lastIndexOfScalar(u8, anchor, std.fs.path.sep)) |idx| idx + 1 else 0;
|
||||
return std.fmt.allocPrint(arena, "{s}{s}", .{ anchor[0..dir_end], name });
|
||||
}
|
||||
|
||||
// ── run ───────────────────────────────────────────────────────
|
||||
|
||||
pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
|
||||
|
|
@ -505,10 +500,10 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
|
|||
if (anchor) |a| {
|
||||
// Accounts - parsed + kept for cross-reference.
|
||||
{
|
||||
const r = checkSrfFile(io, arena, "accounts.srf", try siblingPath(arena, a, "accounts.srf"), .optional, vAccounts);
|
||||
const r = checkSrfFile(io, arena, "accounts.srf", try cli.siblingPath(arena, a, "accounts.srf"), .optional, vAccounts);
|
||||
try checks.append(arena, r);
|
||||
if (r.status == .ok) {
|
||||
const path = try siblingPath(arena, a, "accounts.srf");
|
||||
const path = try cli.siblingPath(arena, a, "accounts.srf");
|
||||
if (std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(16 * 1024 * 1024))) |b| {
|
||||
account_map = analysis.parseAccountsFile(arena, b) catch null;
|
||||
} else |_| {}
|
||||
|
|
@ -516,10 +511,10 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
|
|||
}
|
||||
// Metadata - parsed + kept.
|
||||
{
|
||||
const r = checkSrfFile(io, arena, "metadata.srf", try siblingPath(arena, a, "metadata.srf"), .optional, vMetadata);
|
||||
const r = checkSrfFile(io, arena, "metadata.srf", try cli.siblingPath(arena, a, "metadata.srf"), .optional, vMetadata);
|
||||
try checks.append(arena, r);
|
||||
if (r.status == .ok) {
|
||||
const path = try siblingPath(arena, a, "metadata.srf");
|
||||
const path = try cli.siblingPath(arena, a, "metadata.srf");
|
||||
if (std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(16 * 1024 * 1024))) |b| {
|
||||
class_map = classification.parseClassificationFile(arena, b) catch null;
|
||||
} else |_| {}
|
||||
|
|
@ -527,16 +522,16 @@ pub fn run(ctx: *framework.RunCtx, _: ParsedArgs) !void {
|
|||
}
|
||||
// Transaction log - parsed + kept.
|
||||
{
|
||||
const r = checkSrfFile(io, arena, "transaction_log.srf", try siblingPath(arena, a, "transaction_log.srf"), .optional, vTransfers);
|
||||
const r = checkSrfFile(io, arena, "transaction_log.srf", try cli.siblingPath(arena, a, "transaction_log.srf"), .optional, vTransfers);
|
||||
try checks.append(arena, r);
|
||||
if (r.status == .ok) {
|
||||
const path = try siblingPath(arena, a, "transaction_log.srf");
|
||||
const path = try cli.siblingPath(arena, a, "transaction_log.srf");
|
||||
if (std.Io.Dir.cwd().readFileAlloc(io, path, arena, .limited(16 * 1024 * 1024))) |b| {
|
||||
transfer_log = transaction_log.parseTransactionLogFile(arena, b) catch null;
|
||||
} else |_| {}
|
||||
}
|
||||
}
|
||||
try checks.append(arena, checkSrfFile(io, arena, "projections.srf", try siblingPath(arena, a, "projections.srf"), .optional, validateSrf));
|
||||
try checks.append(arena, checkSrfFile(io, arena, "projections.srf", try cli.siblingPath(arena, a, "projections.srf"), .optional, validateSrf));
|
||||
// imported_values.srf and the snapshots both live under
|
||||
// <portfolio_dir>/history/, NOT directly beside the
|
||||
// portfolio file.
|
||||
|
|
@ -1359,15 +1354,6 @@ test "cross-ref end to end: missing account surfaces as a warn" {
|
|||
try testing.expect(std.mem.indexOf(u8, c.detail, "Sample HSA") != null);
|
||||
}
|
||||
|
||||
test "siblingPath: joins a filename onto the anchor's directory" {
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
const a = arena.allocator();
|
||||
try testing.expectEqualStrings("/home/u/data/accounts.srf", try siblingPath(a, "/home/u/data/portfolio.srf", "accounts.srf"));
|
||||
// Bare filename (no separator) -> sibling is just the name.
|
||||
try testing.expectEqualStrings("accounts.srf", try siblingPath(a, "portfolio.srf", "accounts.srf"));
|
||||
}
|
||||
|
||||
test "validateSrf: accepts a valid stream, rejects a headerless one" {
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue