diff --git a/src/main.zig b/src/main.zig index 88749dc..600a143 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1149,6 +1149,19 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process var counts: SymbolCounts = .{}; var stats: RefreshStats = .{}; + // Symbols whose candles came back WITHOUT a provider call this run: either + // the TTL was still fresh, or a server sync satisfied the request - + // `service.zig` returns `.cached` for both (see the sync at its line 934). + // These are the only symbols a forced refresh in the post-pass sweep can + // help; for every other finding the provider was asked seconds ago and did + // not have the bar, so asking again in the same run is pure quota burn and + // bypasses the very TTL pacing `expiryAfterFetch` exists to get right. + // + // Keys borrow from `portfolio`, whose `deinit` is function-scoped and so + // outlives the sweep below. Block-scoping that deinit is what dangled the + // equivalent keys in `handleDiagnostics`. + var unfetched = std.StringHashMap(void).init(allocator); + defer unfetched.deinit(); var failed_list = std.ArrayList([]const u8).empty; var lagging_list = std.ArrayList([]const u8).empty; var overdue_list = std.ArrayList([]const u8).empty; @@ -1197,6 +1210,11 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process defer result.deinit(); try stdout.print("candles ok ({s})", .{@tagName(result.source)}); stats.candles.hit(result.source == .fetched); + // `== .cached` rather than `!= .fetched` on purpose: should a third + // `Source` ever appear, this defaults to NOT forcing. That failure + // mode leaves a symbol behind, which surfaces via `stuck` and mails + // the operator; the opposite default burns quota in silence. + if (result.source == .cached) try unfetched.put(sym, {}); // Provider-data-lag check: did we end up with the latest bar // the market should have posted by now? A `.lagging` bar is @@ -1215,7 +1233,15 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process }, .overdue => { sym_freshness = .overdue; - log.info("{s}: latest bar {s} overdue past grace window; assuming market closure (no retry)", .{ sym, ds }); + // States the observation, NOT an inference. This used to + // say "assuming market closure (no retry)", which the + // pass cannot know: `candleFreshness` judges this symbol + // against the trading calendar in isolation, and a + // closure moves EVERY symbol together. A single symbol + // nothing refreshes produces the identical reading. The + // corpus question is answered after the pass, by + // `sweepAfterPass`, which can see the peers. + log.info("{s}: latest bar {s} overdue past grace window; no retry this pass", .{ sym, ds }); }, .current => {}, } @@ -1378,14 +1404,27 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process } } + // Sweep AFTER every symbol has been through, because the question it answers + // - is anything behind its peers? - has no answer until the corpus is whole. + const sweep = sweepAfterPass(io, allocator, &svc, config.cache_dir, &symbols, &unfetched, now_s, stdout) catch |err| blk: { + log.warn("post-pass sweep failed: {t}", .{err}); + break :blk SweepOutcome{}; + }; + try stdout.flush(); + const elapsed_ns = std.Io.Timestamp.now(io, .awake).nanoseconds - start_ns; const elapsed_s: u64 = @intCast(@divTrunc(elapsed_ns, std.time.ns_per_s)); - const code = refreshExit(counts.failed, counts.lagging); - const reason = switch (code) { - 0 => "clean", - 75 => "lagging", - else => "failures", - }; + const stuck = if (sweepIsActionable(sweep)) sweep.stuck else 0; + const code = refreshExit(counts.failed, counts.lagging, stuck); + // Distinguishes the two paths to `1`, since the code alone cannot. + const reason = if (counts.failed > 0) + "failures" + else if (stuck > 0) + "stuck behind peers" + else if (code == 75) + "lagging" + else + "clean"; try stdout.print("\nRefresh complete in {d}s (exit {d}: {s})\n", .{ elapsed_s, code, reason }); try stdout.print(" symbols: {d} current, {d} lagging, {d} overdue, {d} failed ({d} total)\n", .{ counts.current, counts.lagging, counts.overdue, counts.failed, symbols.count() }); @@ -1408,16 +1447,316 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process return code; } +/// What the post-pass sweep found and what it managed to fix. +const SweepOutcome = struct { + /// Tracked symbols still further behind their peers than ordinary lag + /// explains, AFTER a forced refresh. These will not clear on the next cron + /// tick, which is why they get their own exit disposition. + stuck: usize = 0, + /// Findings the sweep saw before it acted on anything. + /// + /// Distinct from `attempted` because "nothing was behind" and "things were + /// behind but none were worth re-asking" are different states, and only the + /// first may print a market-closure conclusion. Conflating them would + /// reintroduce the false inference the pass-time log used to make. + found: usize = 0, + /// Symbols the sweep force-refreshed. + attempted: usize = 0, + /// Findings deliberately left alone: the pass already asked the provider for + /// these this run, so a forced re-ask cannot produce a different answer. + skipped: usize = 0, + /// Of those, how many the forced fetch actually brought level with peers. + recovered: usize = 0, + /// Symbols still behind their peers after the retry, INCLUDING those within + /// what ordinary lag explains. Tracked separately from `stuck` because + /// keying the all-clear message on `stuck` alone announced "all clear" while + /// a symbol was still behind - it had merely crossed back inside + /// `max_normal_lag_days`, which is progress, not resolution. + still_behind: usize = 0, + /// Was the sweep able to reach a conclusion at all? False when the cache + /// could not be enumerated, in which case nothing here is a finding. + ran: bool = false, + /// True when no individual symbol is behind its peers but the corpus as a + /// whole sits behind the calendar. That is the shape a market closure makes, + /// and the only shape from which one can honestly be inferred. + corpus_behind: bool = false, + /// The peer reference the conclusion was drawn against, for the operator to + /// check the reasoning rather than take it on faith. + peer_date: ?zfin.Date = null, +}; + +/// Should this run exit non-zero for a gap that will not self-heal? +/// +/// Split from `refreshExit` so the "is a multi-day gap actionable" question is +/// testable without constructing a whole run: `stuck` counts only symbols still +/// past `max_normal_lag_days` after a FORCED refresh, so it excludes both +/// ordinary provider lag (which `75` already covers) and anything a retry fixes. +fn sweepIsActionable(o: SweepOutcome) bool { + return o.ran and o.stuck > 0; +} + +/// Which findings a forced refresh can plausibly help, worst first. +/// +/// Only symbols the pass did NOT already ask the provider about. A symbol whose +/// candles were fetched seconds ago will get the identical answer from an +/// immediate re-ask, so forcing it wastes a request AND bypasses the TTL pacing +/// that exists to space retries out. The case that DOES benefit is a symbol the +/// pass never reached the provider for - a fresh TTL, or a server sync that +/// satisfied the request while serving a copy weeks old. That is precisely the +/// shape of the bug this whole sweep was built for. +/// +/// `far_behind` before `stale` so a rate limit, if one bites mid-sweep, bites the +/// least-behind symbols. Both input lists arrive already sorted worst-first. +/// +/// Pure given the report and the set, which is the point: the "don't waste calls" +/// rule is testable without a provider. +fn selectForForcing( + arena: std.mem.Allocator, + before: zfin.freshness.Report, + unfetched: *const std.StringHashMap(void), +) ![]const []const u8 { + var out = std.ArrayList([]const u8).empty; + for (before.far_behind) |f| { + if (unfetched.contains(f.symbol)) try out.append(arena, f.symbol); + } + for (before.stale) |f| { + if (unfetched.contains(f.symbol)) try out.append(arena, f.symbol); + } + return out.items; +} + +/// Is `symbol` still behind its peers in `report`? +fn isFinding(report: zfin.freshness.Report, symbol: []const u8) bool { + for (report.far_behind) |f| { + if (std.mem.eql(u8, f.symbol, symbol)) return true; + } + for (report.stale) |f| { + if (std.mem.eql(u8, f.symbol, symbol)) return true; + } + return false; +} + +/// Sweep the whole cache after the main pass, force-refresh whatever is behind +/// its peers, then re-check. +/// +/// Why this cannot be folded into the pass: the pass judges each symbol against +/// the trading calendar in isolation (`market.candleFreshness`), which cannot +/// tell a market closure from a single symbol nothing refreshes - both leave a +/// bar sitting past the grace window. A closure moves every symbol together, so +/// "behind its own peers" is disproof of one. That is a corpus question, and the +/// corpus is only complete once every symbol has been through the pass. +/// +/// The forced refresh is the point, not a nicety: a symbol can be behind purely +/// because nothing ever asked for it, and one forced fetch brings it fully +/// current (observed on a watchlist symbol 42 days behind). Reporting the gap +/// without attempting the fix would file a ticket for something the run could +/// have closed itself. +fn sweepAfterPass( + io: std.Io, + allocator: std.mem.Allocator, + svc: *zfin.DataService, + cache_dir: []const u8, + tracked: *const std.StringHashMap(void), + unfetched: *const std.StringHashMap(void), + now_s: i64, + stdout: *std.Io.Writer, +) !SweepOutcome { + // One arena for the whole sweep: it runs once, at the end of a process that + // is about to exit, and the alternative is five separate ownership dances + // across two scans. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var store = zfin.cache.Store.init(io, arena, cache_dir); + const keys = store.cacheKeys(arena) catch |err| { + // Not a finding: an unreadable cache directory means the sweep has no + // opinion, and reporting `stuck = 0` as though it had checked would be + // the same lie the old log line told. + log.warn("post-pass sweep skipped: cannot enumerate cache: {t}", .{err}); + return .{}; + }; + + const before = try zfin.freshness.scan( + arena, + try zfin.freshness.collect(arena, &store, keys, tracked, &.{}), + now_s, + ); + + var out = SweepOutcome{ .ran = true }; + out.peer_date = widestPeerDate(before); + + out.found = before.stale.len + before.far_behind.len; + if (out.found == 0) { + // Nothing is behind its peers. The only remaining question is whether the + // corpus itself has moved, which is what distinguishes "quiet market" from + // "nothing ran" - and it is the ONLY state from which a market closure can + // honestly be inferred. + out.corpus_behind = corpusBehind(before); + try printSweepConclusion(stdout, out, &.{}); + return out; + } + + const forced = try selectForForcing(arena, before, unfetched); + out.attempted = forced.len; + out.skipped = out.found - forced.len; + for (forced) |sym| forceOne(svc, sym); + + // Nothing was forced, so the cache is byte-identical to the pre-sweep scan - + // re-reading it would cost ~40 file reads to learn what we already know, and + // could only differ by disagreeing with itself. + const after = if (forced.len == 0) before else blk: { + // Re-read from disk rather than trusting the fetch's return value: the + // question is what a CLIENT will now be served, and that is whatever + // landed in the cache. + var after_store = zfin.cache.Store.init(io, arena, cache_dir); + const after_keys = after_store.cacheKeys(arena) catch keys; + break :blk try zfin.freshness.scan( + arena, + try zfin.freshness.collect(arena, &after_store, after_keys, tracked, &.{}), + now_s, + ); + }; + + out.still_behind = after.stale.len + after.far_behind.len; + out.stuck = after.far_behind.len; + // Counted by membership, not by subtracting totals. The arithmetic form + // (`attempted - still_behind`) already produced a wrong number once, and + // skipped symbols make it wrong in a second way: they inflate + // `still_behind` without ever having been attempted. + for (forced) |sym| { + if (!isFinding(after, sym)) out.recovered += 1; + } + out.peer_date = widestPeerDate(after) orelse out.peer_date; + // A symbol behind its peers is disproof of a closure, so this stays false + // whenever there are findings - regardless of how the corpus looks. + out.corpus_behind = false; + + try printSweepConclusion(stdout, out, after.far_behind); + return out; +} + +/// Force one symbol's candles, swallowing failures by design: the sweep's job is +/// to report the post-attempt state, and a fetch error here is already visible in +/// the re-scan as "still behind". Logged by name so the cause is not lost. +fn forceOne(svc: *zfin.DataService, symbol: []const u8) void { + if (svc.getCandles(symbol, .{ .force_refresh = true })) |result| { + result.deinit(); + } else |err| { + log.warn("post-pass sweep: forced refresh of {s} failed: {t}", .{ symbol, err }); + } +} + +/// The newest bar any conclusive peer group holds. The reference the sweep's +/// conclusions are measured against. +fn widestPeerDate(report: zfin.freshness.Report) ?zfin.Date { + var newest: ?zfin.Date = null; + for (report.groups) |g| { + if (!g.conclusive()) continue; + const p = g.peer_date orelse continue; + if (newest == null or newest.?.lessThan(p)) newest = p; + } + return newest; +} + +/// Is every conclusive group's own newest bar behind the calendar? +/// +/// Only meaningful when nothing is behind its peers - which the caller enforces. +/// `GroupState.freshness` is the same verdict the fetch gate uses, so this does +/// not re-derive the market calendar. +fn corpusBehind(report: zfin.freshness.Report) bool { + var conclusive: usize = 0; + for (report.groups) |g| { + if (!g.conclusive()) continue; + conclusive += 1; + const f = g.freshness orelse return false; + if (f == .current) return false; + } + return conclusive > 0; +} + +fn printSweepConclusion( + stdout: *std.Io.Writer, + o: SweepOutcome, + stuck: []const zfin.freshness.Finding, +) !void { + if (!o.ran) return; + try stdout.print("\nPost-pass sweep:\n", .{}); + // Keyed on `found`, NOT `attempted`. Those diverged the moment the sweep + // learned to skip symbols the pass had already asked about, and keying the + // closure conclusion on `attempted` would have claimed a market closure while + // symbols were demonstrably behind their peers - the exact false inference + // that was removed from the pass-time log. + if (o.found == 0) { + if (o.corpus_behind) { + // The one case where a closure CAN be inferred, and it is inferred + // from the corpus moving together rather than from one symbol. + try stdout.print(" no symbol is behind its peers, but the whole cache sits behind the\n", .{}); + try stdout.print(" calendar - consistent with a market closure, not a refresh problem\n", .{}); + } else { + try stdout.print(" nothing behind its peers\n", .{}); + } + return; + } + + if (o.attempted == 0) { + // The 5pm case: equities posted, a couple of symbols have not, and the + // pass already asked the provider about them this run. Re-asking cannot + // change the answer, so the sweep does nothing and says why. + try stdout.print(" {d} behind peers; none re-asked - the provider was already queried this pass\n", .{o.found}); + } else if (o.skipped > 0) { + try stdout.print(" {d} behind peers; {d} re-asked, {d} skipped (already queried this pass); {d} now level\n", .{ o.found, o.attempted, o.skipped, o.recovered }); + } else { + try stdout.print(" {d} symbol(s) behind peers -> forced refresh; {d} now level with peers\n", .{ o.attempted, o.recovered }); + } + + // Ordered by severity, NOT by field convenience. An earlier arrangement + // tested `still_behind == 0` first, which would print "all clear" for any + // outcome whose counters disagreed - the one direction this must never fail + // in, since the all-clear is what suppresses the non-zero exit. + if (o.stuck > 0) { + // Named explicitly as non-self-healing, because the whole reason this run + // exits non-zero is to stop cron from looping on it silently. + try stdout.print(" {d} still further behind than lag explains - a retry will not clear these:\n", .{o.stuck}); + for (stuck) |f| { + try stdout.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date }); + } + // Deliberately not naming a cause. The candidates are many and none + // visible from here (see `zfin.freshness.max_normal_lag_days`); + // `zfin diagnose SYMBOL` is the tool that narrows it. + try stdout.print(" run `zfin diagnose SYMBOL` against one of these to narrow it\n", .{}); + return; + } + if (o.still_behind > 0) { + // Partial progress, and saying so matters: these crossed back inside + // `max_normal_lag_days`, so the next pass should close them and this run + // must NOT exit as though a human were needed. + try stdout.print(" {d} still behind, but within what ordinary lag explains - the next pass should close it\n", .{o.still_behind}); + return; + } + try stdout.print(" all clear after the retry\n", .{}); +} + /// Map a refresh run's failure/lag counts to a process exit code: /// 0 - every symbol current and fetched cleanly /// 75 - EX_TEMPFAIL: no hard failures, but at least one symbol's /// just-closed bar hadn't posted yet (provider lag); cron should /// retry shortly -/// 1 - at least one hard failure (fetch error) +/// 1 - at least one hard failure (fetch error), OR a tracked symbol +/// still further behind its peers than lag explains after the +/// post-pass sweep forced a refresh /// Hard failure dominates lag - if anything failed outright that's the /// code the operator needs to act on. -fn refreshExit(fail_count: usize, lag_count: usize) u8 { +/// +/// The stuck-symbol case is deliberately `1` rather than `75`, and this is the +/// distinction that matters: `75` tells cron "retry soon", which is right for an +/// unposted bar and wrong for a multi-day gap. A gap the sweep could not close +/// with a forced fetch will not close on the next tick either, so the run has to +/// mail rather than loop. It shares `1` with a fetch failure because both mean +/// "a human should look"; the summary line names which one it was. +fn refreshExit(fail_count: usize, lag_count: usize, stuck_count: usize) u8 { if (fail_count > 0) return 1; + if (stuck_count > 0) return 1; if (lag_count > 0) return 75; return 0; } @@ -1644,12 +1983,101 @@ test "isPlausibleSymbol" { try std.testing.expect(!isPlausibleSymbol("ABCDEFGHIJKLMNOPQ")); // 17 chars, too long } -test "refreshExit: hard failure dominates, then lag, else clean" { - try std.testing.expectEqual(@as(u8, 0), refreshExit(0, 0)); - try std.testing.expectEqual(@as(u8, 75), refreshExit(0, 3)); - try std.testing.expectEqual(@as(u8, 1), refreshExit(2, 0)); +test "refreshExit: hard failure dominates, then stuck, then lag, else clean" { + try std.testing.expectEqual(@as(u8, 0), refreshExit(0, 0, 0)); + try std.testing.expectEqual(@as(u8, 75), refreshExit(0, 3, 0)); + try std.testing.expectEqual(@as(u8, 1), refreshExit(2, 0, 0)); // A hard failure outranks lag. - try std.testing.expectEqual(@as(u8, 1), refreshExit(1, 5)); + try std.testing.expectEqual(@as(u8, 1), refreshExit(1, 5, 0)); + + // A symbol still behind its peers after a FORCED refresh exits 1, not 75. + // This is the whole point of the third argument: 75 means EX_TEMPFAIL, which + // tells cron to retry soon - correct for an unposted bar, wrong for a + // multi-day gap that a forced fetch already failed to close. Retrying that on + // a schedule loops silently forever, which is how SPCX went 43 days unnoticed. + try std.testing.expectEqual(@as(u8, 1), refreshExit(0, 0, 1)); + // Stuck outranks lag: the actionable finding wins over the retryable one. + try std.testing.expectEqual(@as(u8, 1), refreshExit(0, 9, 1)); + // But a hard failure still outranks stuck - it is the more proximate problem + // and may well be the CAUSE of the gap. + try std.testing.expectEqual(@as(u8, 1), refreshExit(4, 0, 2)); +} + +test "sweepIsActionable: only a sweep that actually ran can be a finding" { + // A sweep that could not enumerate the cache reports `stuck = 0`, and reading + // that as "nothing is behind" would repeat the exact mistake the old + // market-closure log line made: asserting a conclusion from missing evidence. + // `ran` is what separates "checked and found nothing" from "did not check". + try std.testing.expect(!sweepIsActionable(.{ .ran = false, .stuck = 3 })); + try std.testing.expect(!sweepIsActionable(.{ .ran = true, .stuck = 0 })); + try std.testing.expect(sweepIsActionable(.{ .ran = true, .stuck = 1 })); + // The zero value must never be actionable - it is what the error paths return. + try std.testing.expect(!sweepIsActionable(.{})); +} + +test "corpusBehind: a closure is only inferable when every group moved together" { + const d = zfin.Date.fromYmd(2026, 8, 12); + const empty: []zfin.freshness.Finding = &.{}; + + // Two conclusive groups, both overdue, nothing behind its peers. This is the + // one shape from which a market closure can honestly be read. + var groups = [_]zfin.freshness.GroupState{ + .{ .kind = .equity, .peer_date = d, .freshness = .overdue, .dated = 9 }, + .{ .kind = .mutual_fund, .peer_date = d, .freshness = .overdue, .dated = 4 }, + }; + var r = zfin.freshness.Report{ + .stale = empty, + .far_behind = empty, + .orphans = &.{}, + .missing = &.{}, + .groups = &groups, + }; + try std.testing.expect(corpusBehind(r)); + + // One group current: the market plainly was not closed. + groups[1].freshness = .current; + try std.testing.expect(!corpusBehind(r)); + + // Unknown freshness is not evidence of a closure. Absence of information must + // not become a conclusion. + groups[1].freshness = null; + try std.testing.expect(!corpusBehind(r)); + + // No conclusive group at all - a single cached symbol per kind - concludes + // nothing rather than vacuously true. + var lonely = [_]zfin.freshness.GroupState{ + .{ .kind = .equity, .peer_date = d, .freshness = .overdue, .dated = 1 }, + }; + r.groups = &lonely; + try std.testing.expect(!corpusBehind(r)); + r.groups = &.{}; + try std.testing.expect(!corpusBehind(r)); +} + +test "widestPeerDate: the newest conclusive group wins, inconclusive ignored" { + const aug12 = zfin.Date.fromYmd(2026, 8, 12); + const aug11 = zfin.Date.fromYmd(2026, 8, 11); + const jun01 = zfin.Date.fromYmd(2026, 6, 1); + var groups = [_]zfin.freshness.GroupState{ + .{ .kind = .equity, .peer_date = aug11, .freshness = null, .dated = 5 }, + .{ .kind = .mutual_fund, .peer_date = aug12, .freshness = null, .dated = 3 }, + }; + var r = zfin.freshness.Report{ + .stale = &.{}, + .far_behind = &.{}, + .orphans = &.{}, + .missing = &.{}, + .groups = &groups, + }; + try std.testing.expectEqual(@as(?zfin.Date, aug12), widestPeerDate(r)); + + // An inconclusive group's date must not become the reference - with one cached + // symbol its "peer date" is just its own bar. + groups[1] = .{ .kind = .mutual_fund, .peer_date = jun01, .freshness = null, .dated = 1 }; + try std.testing.expectEqual(@as(?zfin.Date, aug11), widestPeerDate(r)); + + r.groups = &.{}; + try std.testing.expectEqual(@as(?zfin.Date, null), widestPeerDate(r)); } test "printStatRow aligns with the summary-table header" { @@ -1804,3 +2232,253 @@ test "collectRefreshSymbols: keys stay readable while the source lots live" { try std.testing.expect(set.contains("AMZN")); try std.testing.expectEqual(@as(u32, 1), set.count()); } + +test "printSweepConclusion: a closure is claimed only when the corpus moved together" { + const a = std.testing.allocator; + + // Nothing behind peers, corpus itself behind: the ONE case where a closure is + // a legitimate inference, and it must say so from the corpus, not one symbol. + { + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ .ran = true, .corpus_behind = true }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "not a refresh problem") != null); + } + + // Nothing behind peers and the corpus is current: no closure claim at all. + // The old pass-time log asserted closure from a single symbol's calendar + // position; nothing may reintroduce that from an empty finding list. + { + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ .ran = true, .corpus_behind = false }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "nothing behind its peers") != null); + } + + // A sweep that never ran prints nothing - it has no opinion to report. + { + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{}, &.{}); + try std.testing.expectEqual(@as(usize, 0), w.written().len); + } +} + +test "printSweepConclusion: a stuck symbol is named and marked non-self-healing" { + const a = std.testing.allocator; + const stuck = [_]zfin.freshness.Finding{.{ + .symbol = "SPCX", + .kind = .equity, + .last_date = zfin.Date.fromYmd(2026, 6, 29), + .peer_date = zfin.Date.fromYmd(2026, 8, 12), + .days_behind = 44, + }}; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 3, + .attempted = 3, + .recovered = 2, + .still_behind = 1, + .stuck = 1, + }, &stuck); + const s = w.written(); + + // The symbol and both dates, so the operator can check the reasoning rather + // than trust the verdict. + try std.testing.expect(std.mem.indexOf(u8, s, "SPCX") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "2026-06-29") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "2026-08-12") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "44d behind") != null); + // The recovery tally, so a partial success is not read as total failure. + try std.testing.expect(std.mem.indexOf(u8, s, "2 now level with peers") != null); + // Says a retry will NOT help - the justification for exiting 1 over 75. + try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") != null); + // Points at the tool that narrows a cause instead of guessing one. + try std.testing.expect(std.mem.indexOf(u8, s, "zfin diagnose") != null); + // And must NOT guess. `freshness.max_normal_lag_days` documents that the + // candidates are many and none visible from here. + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); +} + +test "printSweepConclusion: partial progress is not an all-clear, and not a page" { + // The bug this pins, found by running it: `all clear after the retry` printed + // whenever `stuck == 0`, ignoring symbols still behind by a day or two. A + // forced refresh that drags a symbol from 53 days behind to 1 day behind has + // made progress and resolved nothing, and the operator must be able to tell + // those apart from the output alone. + const a = std.testing.allocator; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 1, + .attempted = 1, + .recovered = 0, + .still_behind = 1, + .stuck = 0, + }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "within what ordinary lag explains") != null); + // Must not claim non-recovery is permanent: that language belongs to `stuck`, + // which is what exits 1. + try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") == null); + + // Genuinely resolved: every attempted symbol is level again. + var w2: std.Io.Writer.Allocating = .init(a); + defer w2.deinit(); + try printSweepConclusion(&w2.writer, .{ + .ran = true, + .found = 2, + .attempted = 2, + .recovered = 2, + .still_behind = 0, + .stuck = 0, + }, &.{}); + try std.testing.expect(std.mem.indexOf(u8, w2.written(), "all clear") != null); +} + +test "printSweepConclusion: disagreeing counters never produce a false all-clear" { + // Defence in the one direction that matters. `stuck > 0` with a `still_behind` + // that failed to keep up is a bug in the caller, but the output must degrade + // to the LOUD reading, never the quiet one - the all-clear is what suppresses + // the non-zero exit, so a false all-clear is silent data rot. + const a = std.testing.allocator; + const stuck = [_]zfin.freshness.Finding{.{ + .symbol = "SPCX", + .kind = .equity, + .last_date = zfin.Date.fromYmd(2026, 6, 29), + .peer_date = zfin.Date.fromYmd(2026, 8, 12), + .days_behind = 44, + }}; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 1, + .attempted = 1, + .recovered = 0, + .still_behind = 0, // deliberately inconsistent with `stuck` + .stuck = 1, + }, &stuck); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "SPCX") != null); +} + +/// Build a Report from findings, for the pure decision tests. +fn testReport( + stale: []zfin.freshness.Finding, + far: []zfin.freshness.Finding, +) zfin.freshness.Report { + return .{ .stale = stale, .far_behind = far, .orphans = &.{}, .missing = &.{}, .groups = &.{} }; +} + +fn testFinding(symbol: []const u8, days: i64) zfin.freshness.Finding { + return .{ + .symbol = symbol, + .kind = .equity, + .last_date = zfin.Date.fromYmd(2026, 8, 12).addDays(@intCast(-days)), + .peer_date = zfin.Date.fromYmd(2026, 8, 12), + .days_behind = days, + }; +} + +test "selectForForcing: a symbol the pass already fetched is not re-asked" { + // THE 5pm CASE. 13 equities post, 2 do not. Those 2 had their TTL lapse, so + // the pass made a real provider call and got nothing newer. Re-asking seconds + // later cannot change the answer: it burns quota (50 req/hr on the free Tiingo + // plan) and bypasses the TTL pacing `expiryAfterFetch` exists to enforce. + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var stale = [_]zfin.freshness.Finding{ testFinding("MSFT", 1), testFinding("NKE", 1) }; + const report = testReport(&stale, &.{}); + + // Empty set: the pass fetched everything, so nothing is worth re-asking. + var none = std.StringHashMap(void).init(a); + try std.testing.expectEqual(@as(usize, 0), (try selectForForcing(a, report, &none)).len); +} + +test "selectForForcing: a symbol the pass never fetched IS re-asked" { + // The SPCX case, and the reason the sweep exists. A server sync stamped a + // future `#!expires=`, so the pass returned `.cached` without ever reaching a + // provider - while serving a copy six weeks old. Forcing is the only thing + // that moves it. + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var far = [_]zfin.freshness.Finding{testFinding("SPCX", 44)}; + var stale = [_]zfin.freshness.Finding{testFinding("NKE", 1)}; + const report = testReport(&stale, &far); + + var cached = std.StringHashMap(void).init(a); + try cached.put("SPCX", {}); + try cached.put("NKE", {}); + const sel = try selectForForcing(a, report, &cached); + try std.testing.expectEqual(@as(usize, 2), sel.len); + // far_behind first, so a mid-sweep rate limit bites the least-behind symbol. + try std.testing.expectEqualStrings("SPCX", sel[0]); + try std.testing.expectEqualStrings("NKE", sel[1]); +} + +test "selectForForcing: mixed - only the unfetched half is re-asked" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var far = [_]zfin.freshness.Finding{ testFinding("SPCX", 44), testFinding("ORC42", 30) }; + var stale = [_]zfin.freshness.Finding{ testFinding("MSFT", 1), testFinding("NKE", 2) }; + const report = testReport(&stale, &far); + + var cached = std.StringHashMap(void).init(a); + try cached.put("SPCX", {}); // TTL-held, worth forcing + try cached.put("NKE", {}); // TTL-held, worth forcing + const sel = try selectForForcing(a, report, &cached); + try std.testing.expectEqual(@as(usize, 2), sel.len); + try std.testing.expectEqualStrings("SPCX", sel[0]); + try std.testing.expectEqualStrings("NKE", sel[1]); +} + +test "isFinding: consulted across BOTH lists" { + // An earlier shape of the recovered-count checked only one list, which + // credited recovery to symbols that had merely crossed from far_behind into + // stale - still behind, reported as fixed. + var far = [_]zfin.freshness.Finding{testFinding("SPCX", 44)}; + var stale = [_]zfin.freshness.Finding{testFinding("NKE", 1)}; + const report = testReport(&stale, &far); + try std.testing.expect(isFinding(report, "SPCX")); + try std.testing.expect(isFinding(report, "NKE")); + try std.testing.expect(!isFinding(report, "AMZN")); +} + +test "printSweepConclusion: findings with nothing re-asked is not a closure" { + // The 5pm output. Symbols ARE behind, so no closure may be claimed, and the + // run must explain why it did nothing rather than looking like it missed them. + const a = std.testing.allocator; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 2, + .attempted = 0, + .skipped = 2, + .still_behind = 2, + .stuck = 0, + }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "already queried this pass") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "within what ordinary lag explains") != null); + // Must not read as resolved - the symbols are still behind. + try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); +}