From c15ea92b0d1fc707bc4e5500fdd03d6bf4ff1d18 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 19 Aug 2026 13:53:50 -0700 Subject: [PATCH] fix forward-announced ex-dates masking real restatement work --- src/cache/freshness.zig | 189 +++++++++++++++++++++++++++++--------- src/commands/diagnose.zig | 4 +- src/service.zig | 23 +++-- 3 files changed, 164 insertions(+), 52 deletions(-) diff --git a/src/cache/freshness.zig b/src/cache/freshness.zig index 7e1bbd3..c53634e 100644 --- a/src/cache/freshness.zig +++ b/src/cache/freshness.zig @@ -47,6 +47,12 @@ //! means the series must be refetched is domain policy, and //! `store.zig`'s own `updateCandleMeta` doc records the same boundary //! for market-clock knowledge ("owned by the caller"). +//! +//! Note which half owns the "has it actually gone ex yet?" filter: +//! `newestCorporateAction` does, because that is a question about which +//! actions are *relevant*, and relevance is selection. Putting it on the +//! verdict instead is what let one forward-announced dividend mask every +//! older unapplied one. const std = @import("std"); const Date = @import("../Date.zig"); @@ -638,16 +644,38 @@ test "collect-style exclusion: an excluded symbol produces no finding at all" { // two functions turn that into a verdict, split the same way as // `collect` / `scan`: one gathers from disk, one decides. -/// Newest corporate-action ex-date cached for a symbol, or null when no -/// dividends or splits are on disk. +/// Newest corporate-action ex-date cached for a symbol that has already +/// gone ex as of `through`, or null when there is no such action. /// /// Splits count as much as dividends here, and arguably more: raw /// `close` is not split-adjusted (hence `split.cumulativeSplitRatio`), /// so an unapplied 2:1 split leaves older `adj_close` values wrong by /// 50% rather than by a quarter's yield. /// +/// `through` is normally `CandleMeta.last_date`, the newest bar held. +/// Actions dated after it are skipped, and that bound is load-bearing +/// twice over: +/// +/// - **Convergence.** A declared-but-not-yet-ex distribution is +/// reflected in no provider's adjustment series, so treating it as +/// something to catch up to would refetch on every pass forever. +/// - **Not masking real work.** The bound belongs here, on the +/// selection, rather than on the verdict. Bounding the verdict +/// instead - "is the newest action of all still in the future? then +/// nothing to do" - lets a single forward announcement hide every +/// older unapplied action behind it. That shipped, and it left NKE +/// permanently unable to restate: it announces roughly a quarter +/// ahead, so its newest cached ex-date is essentially always in the +/// future, which masked a distribution that had genuinely gone ex +/// two months earlier. +/// /// Allocates only transiently - the returned `Date` borrows nothing. -pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, symbol: []const u8) ?Date { +pub fn newestCorporateAction( + allocator: std.mem.Allocator, + store: *cache.Store, + symbol: []const u8, + through: Date, +) ?Date { var newest: ?Date = null; // `CacheResult` has no deinit - the caller owns `data`. Dividends @@ -656,12 +684,14 @@ pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, if (store.read(allocator, Dividend, symbol, null, .any)) |r| { defer Dividend.freeSlice(allocator, r.data); for (r.data) |d| { + if (through.lessThan(d.ex_date)) continue; if (newest == null or newest.?.lessThan(d.ex_date)) newest = d.ex_date; } } if (store.read(allocator, Split, symbol, null, .any)) |r| { defer allocator.free(r.data); for (r.data) |sp| { + if (through.lessThan(sp.date)) continue; if (newest == null or newest.?.lessThan(sp.date)) newest = sp.date; } } @@ -674,58 +704,40 @@ pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, /// /// - `adj_basis`: `CandleMeta.adj_basis` - the bar date through which /// the cached `adj_close` values reflect corporate actions. -/// - `last_bar`: `CandleMeta.last_date` - the newest bar held. -/// - `newest_action`: from `newestCorporateAction`, or null for a -/// symbol with no cached dividends or splits. +/// - `newest_ex`: from `newestCorporateAction`, which has already +/// discarded anything not yet ex. Null for a symbol with no +/// applicable cached dividends or splits. /// -/// The `last_bar` bound is what keeps this convergent. A distribution -/// announced with a future ex-date is not yet reflected in *any* -/// provider's adjustment series, so treating it as stale would refetch -/// on every pass and never settle. +/// Takes no `last_bar`: the not-yet-ex bound lives in +/// `newestCorporateAction` so that it cannot be applied to the wrong +/// end of the comparison. See that function for what happened when it +/// was applied here instead. /// -/// Pure: three dates in, bool out. -pub fn adjustmentBasisStale(adj_basis: Date, last_bar: Date, newest_action: ?Date) bool { - const newest = newest_action orelse return false; - if (last_bar.lessThan(newest)) return false; +/// Pure: two dates in, bool out. +pub fn adjustmentBasisStale(adj_basis: Date, newest_ex: ?Date) bool { + const newest = newest_ex orelse return false; return adj_basis.lessThan(newest); } -test "adjustmentBasisStale: no corporate action means nothing to restate" { +test "adjustmentBasisStale: no applicable action means nothing to restate" { // A non-payer has no ex-date to exceed the basis, so it must never // escalate - not even with the epoch sentinel a legacy cache parses to. - try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), Date.fromYmd(2026, 8, 14), null)); - try testing.expect(!adjustmentBasisStale(Date.epoch, Date.fromYmd(2026, 8, 14), null)); + try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), null)); + try testing.expect(!adjustmentBasisStale(Date.epoch, null)); } test "adjustmentBasisStale: action behind the basis is stale" { - const last_bar = Date.fromYmd(2026, 8, 14); const ex = Date.fromYmd(2026, 6, 18); - // Basis at the newest bar already covers the ex-date. - try testing.expect(!adjustmentBasisStale(last_bar, last_bar, ex)); + // Basis past the ex-date already covers it. + try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), ex)); // Basis exactly at the ex-date covers it too (not `lessThan`). - try testing.expect(!adjustmentBasisStale(ex, last_bar, ex)); + try testing.expect(!adjustmentBasisStale(ex, ex)); // Basis behind the ex-date: the bars in between were never marked down. - try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), last_bar, ex)); + try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), ex)); // The legacy-cache case: sentinel basis on a dividend payer. This is // the shape every pre-adj_basis cache lands in. - try testing.expect(adjustmentBasisStale(Date.epoch, last_bar, ex)); -} - -test "adjustmentBasisStale: a not-yet-ex action is not stale" { - // Ex-date beyond the newest bar we hold. No provider has applied it - // either, so escalating would never converge. - try testing.expect(!adjustmentBasisStale( - Date.fromYmd(2026, 5, 1), - Date.fromYmd(2026, 8, 14), - Date.fromYmd(2026, 9, 17), - )); - // Boundary: an ex-date exactly on the newest bar IS covered. - try testing.expect(adjustmentBasisStale( - Date.fromYmd(2026, 5, 1), - Date.fromYmd(2026, 8, 14), - Date.fromYmd(2026, 8, 14), - )); + try testing.expect(adjustmentBasisStale(Date.epoch, ex)); } test "newestCorporateAction: takes the max across dividends and splits" { @@ -737,23 +749,114 @@ test "newestCorporateAction: takes the max across dividends and splits" { defer a.free(dir_path); var store = cache.Store.init(io, a, dir_path); + const through = Date.fromYmd(2026, 8, 14); // Nothing cached at all. - try testing.expect(newestCorporateAction(a, &store, "SMPL") == null); + try testing.expect(newestCorporateAction(a, &store, "SMPL", through) == null); var divs = [_]Dividend{ .{ .ex_date = Date.fromYmd(2026, 3, 20), .amount = 1.79 }, .{ .ex_date = Date.fromYmd(2026, 6, 18), .amount = 1.90 }, }; store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends }); - try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 6, 18))); + try testing.expect(newestCorporateAction(a, &store, "SMPL", through).?.eql(Date.fromYmd(2026, 6, 18))); // A later split must win over the later dividend. var splits = [_]Split{.{ .date = Date.fromYmd(2026, 7, 1), .numerator = 2, .denominator = 1 }}; store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits }); - try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 7, 1))); + try testing.expect(newestCorporateAction(a, &store, "SMPL", through).?.eql(Date.fromYmd(2026, 7, 1))); // Splits alone (different symbol) still resolve. store.write(Split, "SMPLB", splits[0..], .{ .seconds = cache.Ttl.splits }); - try testing.expect(newestCorporateAction(a, &store, "SMPLB").?.eql(Date.fromYmd(2026, 7, 1))); + try testing.expect(newestCorporateAction(a, &store, "SMPLB", through).?.eql(Date.fromYmd(2026, 7, 1))); +} + +test "newestCorporateAction: a forward-announced action does not mask an older unapplied one" { + // The regression. NKE announces roughly a quarter ahead, so its + // newest cached ex-date is essentially always in the future. Bounding + // the *verdict* on "is the newest action still in the future?" made + // that one announcement hide a distribution that had gone ex two + // months earlier, and NKE could never restate. The bound belongs on + // the selection, so the announcement is skipped and the older + // already-ex action is still found. + const io = testing.io; + const a = testing.allocator; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir_path); + + var store = cache.Store.init(io, a, dir_path); + const last_bar = Date.fromYmd(2026, 8, 18); + + // NKE's exact shape: applied through 2026-03-02, 2026-06-01 went ex + // and was never applied, 2026-09-01 declared but not yet ex. + var divs = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2026, 9, 1), .amount = 0.41 }, + .{ .ex_date = Date.fromYmd(2026, 6, 1), .amount = 0.41 }, + .{ .ex_date = Date.fromYmd(2026, 3, 2), .amount = 0.41 }, + }; + store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends }); + + // The future announcement is skipped; the newest already-ex wins. + const newest = newestCorporateAction(a, &store, "SMPL", last_bar); + try testing.expect(newest != null); + try testing.expect(newest.?.eql(Date.fromYmd(2026, 6, 1))); + + // And the verdict is therefore "stale" for a legacy sentinel basis. + try testing.expect(adjustmentBasisStale(Date.epoch, newest)); + // ...and for a basis covering 2026-03-02 but not 2026-06-01. + try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 4, 1), newest)); + // ...but not once the basis has caught up. + try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 17), newest)); +} + +test "newestCorporateAction: a declared-but-not-effective split does not mask either" { + const io = testing.io; + const a = testing.allocator; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir_path); + + var store = cache.Store.init(io, a, dir_path); + const last_bar = Date.fromYmd(2026, 8, 18); + + var splits = [_]Split{ + .{ .date = Date.fromYmd(2026, 10, 1), .numerator = 2, .denominator = 1 }, + .{ .date = Date.fromYmd(2026, 5, 1), .numerator = 3, .denominator = 1 }, + }; + store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits }); + + const newest = newestCorporateAction(a, &store, "SMPL", last_bar); + try testing.expect(newest.?.eql(Date.fromYmd(2026, 5, 1))); + try testing.expect(adjustmentBasisStale(Date.epoch, newest)); +} + +test "newestCorporateAction: convergence - an only-future action is not actionable" { + // The property the bad bound was reaching for, preserved. A + // declared-but-not-yet-ex action is in no provider's adjustment + // series, so treating it as something to catch up to would refetch + // the full history on every pass, forever. + const io = testing.io; + const a = testing.allocator; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir_path); + + var store = cache.Store.init(io, a, dir_path); + + var divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 9, 17), .amount = 1.9 }}; + store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends }); + + const last_bar = Date.fromYmd(2026, 8, 18); + try testing.expect(newestCorporateAction(a, &store, "SMPL", last_bar) == null); + try testing.expect(!adjustmentBasisStale(Date.epoch, newestCorporateAction(a, &store, "SMPL", last_bar))); + + // Boundary: once it goes ex - `through` reaching the ex-date - it + // becomes actionable on that very bar. + const on_ex = Date.fromYmd(2026, 9, 17); + try testing.expect(newestCorporateAction(a, &store, "SMPL", on_ex).?.eql(on_ex)); + try testing.expect(adjustmentBasisStale(Date.epoch, newestCorporateAction(a, &store, "SMPL", on_ex))); } diff --git a/src/commands/diagnose.zig b/src/commands/diagnose.zig index de97471..922cf00 100644 --- a/src/commands/diagnose.zig +++ b/src/commands/diagnose.zig @@ -254,8 +254,8 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void { // bars behind it - total returns then read low by roughly the // missed yield. Report it here because the symptom (a slightly // low 1Y total return) is otherwise invisible. - if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| { - if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) { + if (freshness.newestCorporateAction(arena, &store, symbol, m.meta.last_date)) |newest_action| { + if (freshness.adjustmentBasisStale(m.meta.adj_basis, newest_action)) { try out.print( "adj basis {f} - STALE, {f} went ex behind it; total returns read low until restated\n", .{ m.meta.adj_basis, newest_action }, diff --git a/src/service.zig b/src/service.zig index 5883dc7..253505d 100644 --- a/src/service.zig +++ b/src/service.zig @@ -843,10 +843,21 @@ pub const DataService = struct { next.provider = provider; next.fail_count = 0; + // A provider change and a cleared backoff are independent facts, + // and conflating them hid the first one. This used to log only + // when clearing an armed backoff - but the case that mattered + // was a legacy cache carrying `provider = .yahoo` with no + // backoff at all, which is every symbol that had drifted off + // Tiingo before `tiingo_retry_after_s` existed. Twenty-one + // symbols converted back to Tiingo without a single line. + if (meta.provider != provider) { + log.info("{s}: candle provider {t} -> {t}", .{ symbol, meta.provider, provider }); + } + switch (coverage) { .covered => { if (meta.tiingo_retry_after_s != 0) { - log.info("{s}: provider converted {t} -> tiingo, clearing Tiingo backoff", .{ symbol, meta.provider }); + log.info("{s}: Tiingo serving again, clearing backoff", .{symbol}); } next.tiingo_retry_after_s = 0; }, @@ -1147,9 +1158,9 @@ pub const DataService = struct { // see a same-dated file whose historical adj_close is // stale, so the basis has to be re-checked here. const synced = if (s.readCandleMeta(symbol)) |sm| sm.meta else m; - const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol); + const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol, synced.last_date); if (s.isCandleMetaFresh(symbol) and - !freshness.adjustmentBasisStale(synced.adj_basis, synced.last_date, synced_action)) + !freshness.adjustmentBasisStale(synced.adj_basis, synced_action)) { log.debug("{s}: candles synced from server and fresh", .{symbol}); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| @@ -1180,8 +1191,7 @@ pub const DataService = struct { // tax the hot portfolio-pricing path for nothing. if (freshness.adjustmentBasisStale( m.adj_basis, - m.last_date, - freshness.newestCorporateAction(self.allocator, &s, symbol), + freshness.newestCorporateAction(self.allocator, &s, symbol, m.last_date), )) { log.info("{s}: restating full history (adj_basis {f} predates a corporate action)", .{ symbol, m.adj_basis }); if (self.refetchFullHistory(symbol, today, now_s, now_s < m.tiingo_retry_after_s)) |candles| { @@ -4301,8 +4311,7 @@ test "getCandles offline never escalates a stale adjustment basis" { store.updateCandleMeta("SMPL", meta, 1); // expiry in the past => stale try std.testing.expect(freshness.adjustmentBasisStale( meta.adj_basis, - meta.last_date, - freshness.newestCorporateAction(allocator, &store, "SMPL"), + freshness.newestCorporateAction(allocator, &store, "SMPL", meta.last_date), )); svc.panic_on_network_attempt = true;