fix forward-announced ex-dates masking real restatement work

src/cache/freshness.zig

@@ -47,6 +47,12 @@
 //! means the series must be refetched is domain policy, and
 //! `store.zig`'s own `updateCandleMeta` doc records the same boundary
 //! for market-clock knowledge ("owned by the caller").
+//!
+//! Note which half owns the "has it actually gone ex yet?" filter:
+//! `newestCorporateAction` does, because that is a question about which
+//! actions are *relevant*, and relevance is selection. Putting it on the
+//! verdict instead is what let one forward-announced dividend mask every
+//! older unapplied one.
 
 const std = @import("std");
 const Date = @import("../Date.zig");
@@ -638,16 +644,38 @@ test "collect-style exclusion: an excluded symbol produces no finding at all" {
 // two functions turn that into a verdict, split the same way as
 // `collect` / `scan`: one gathers from disk, one decides.
 
-/// Newest corporate-action ex-date cached for a symbol, or null when no
+/// Newest corporate-action ex-date cached for a symbol that has already
-/// dividends or splits are on disk.
+/// gone ex as of `through`, or null when there is no such action.
 ///
 /// Splits count as much as dividends here, and arguably more: raw
 /// `close` is not split-adjusted (hence `split.cumulativeSplitRatio`),
 /// so an unapplied 2:1 split leaves older `adj_close` values wrong by
 /// 50% rather than by a quarter's yield.
 ///
+/// `through` is normally `CandleMeta.last_date`, the newest bar held.
+/// Actions dated after it are skipped, and that bound is load-bearing
+/// twice over:
+///
+///   - **Convergence.** A declared-but-not-yet-ex distribution is
+///     reflected in no provider's adjustment series, so treating it as
+///     something to catch up to would refetch on every pass forever.
+///   - **Not masking real work.** The bound belongs here, on the
+///     selection, rather than on the verdict. Bounding the verdict
+///     instead - "is the newest action of all still in the future? then
+///     nothing to do" - lets a single forward announcement hide every
+///     older unapplied action behind it. That shipped, and it left NKE
+///     permanently unable to restate: it announces roughly a quarter
+///     ahead, so its newest cached ex-date is essentially always in the
+///     future, which masked a distribution that had genuinely gone ex
+///     two months earlier.
+///
 /// Allocates only transiently - the returned `Date` borrows nothing.
-pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, symbol: []const u8) ?Date {
+pub fn newestCorporateAction(
+    allocator: std.mem.Allocator,
+    store: *cache.Store,
+    symbol: []const u8,
+    through: Date,
+) ?Date {
     var newest: ?Date = null;
 
     // `CacheResult` has no deinit - the caller owns `data`. Dividends
@@ -656,12 +684,14 @@ pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store,
     if (store.read(allocator, Dividend, symbol, null, .any)) |r| {
         defer Dividend.freeSlice(allocator, r.data);
         for (r.data) |d| {
+            if (through.lessThan(d.ex_date)) continue;
             if (newest == null or newest.?.lessThan(d.ex_date)) newest = d.ex_date;
         }
     }
     if (store.read(allocator, Split, symbol, null, .any)) |r| {
         defer allocator.free(r.data);
         for (r.data) |sp| {
+            if (through.lessThan(sp.date)) continue;
             if (newest == null or newest.?.lessThan(sp.date)) newest = sp.date;
         }
     }
@@ -674,58 +704,40 @@ pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store,
 ///
 /// - `adj_basis`: `CandleMeta.adj_basis` - the bar date through which
 ///   the cached `adj_close` values reflect corporate actions.
-/// - `last_bar`: `CandleMeta.last_date` - the newest bar held.
+/// - `newest_ex`: from `newestCorporateAction`, which has already
-/// - `newest_action`: from `newestCorporateAction`, or null for a
+///   discarded anything not yet ex. Null for a symbol with no
-///   symbol with no cached dividends or splits.
+///   applicable cached dividends or splits.
 ///
-/// The `last_bar` bound is what keeps this convergent. A distribution
+/// Takes no `last_bar`: the not-yet-ex bound lives in
-/// announced with a future ex-date is not yet reflected in *any*
+/// `newestCorporateAction` so that it cannot be applied to the wrong
-/// provider's adjustment series, so treating it as stale would refetch
+/// end of the comparison. See that function for what happened when it
-/// on every pass and never settle.
+/// was applied here instead.
 ///
-/// Pure: three dates in, bool out.
+/// Pure: two dates in, bool out.
-pub fn adjustmentBasisStale(adj_basis: Date, last_bar: Date, newest_action: ?Date) bool {
+pub fn adjustmentBasisStale(adj_basis: Date, newest_ex: ?Date) bool {
-    const newest = newest_action orelse return false;
+    const newest = newest_ex orelse return false;
-    if (last_bar.lessThan(newest)) return false;
     return adj_basis.lessThan(newest);
 }
 
-test "adjustmentBasisStale: no corporate action means nothing to restate" {
+test "adjustmentBasisStale: no applicable action means nothing to restate" {
     // A non-payer has no ex-date to exceed the basis, so it must never
     // escalate - not even with the epoch sentinel a legacy cache parses to.
-    try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), Date.fromYmd(2026, 8, 14), null));
+    try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), null));
-    try testing.expect(!adjustmentBasisStale(Date.epoch, Date.fromYmd(2026, 8, 14), null));
+    try testing.expect(!adjustmentBasisStale(Date.epoch, null));
 }
 
 test "adjustmentBasisStale: action behind the basis is stale" {
-    const last_bar = Date.fromYmd(2026, 8, 14);
     const ex = Date.fromYmd(2026, 6, 18);
 
-    // Basis at the newest bar already covers the ex-date.
+    // Basis past the ex-date already covers it.
-    try testing.expect(!adjustmentBasisStale(last_bar, last_bar, ex));
+    try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), ex));
     // Basis exactly at the ex-date covers it too (not `lessThan`).
-    try testing.expect(!adjustmentBasisStale(ex, last_bar, ex));
+    try testing.expect(!adjustmentBasisStale(ex, ex));
     // Basis behind the ex-date: the bars in between were never marked down.
-    try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), last_bar, ex));
+    try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), ex));
     // The legacy-cache case: sentinel basis on a dividend payer. This is
     // the shape every pre-adj_basis cache lands in.
-    try testing.expect(adjustmentBasisStale(Date.epoch, last_bar, ex));
+    try testing.expect(adjustmentBasisStale(Date.epoch, ex));
-}
-
-test "adjustmentBasisStale: a not-yet-ex action is not stale" {
-    // Ex-date beyond the newest bar we hold. No provider has applied it
-    // either, so escalating would never converge.
-    try testing.expect(!adjustmentBasisStale(
-        Date.fromYmd(2026, 5, 1),
-        Date.fromYmd(2026, 8, 14),
-        Date.fromYmd(2026, 9, 17),
-    ));
-    // Boundary: an ex-date exactly on the newest bar IS covered.
-    try testing.expect(adjustmentBasisStale(
-        Date.fromYmd(2026, 5, 1),
-        Date.fromYmd(2026, 8, 14),
-        Date.fromYmd(2026, 8, 14),
-    ));
 }
 
 test "newestCorporateAction: takes the max across dividends and splits" {
@@ -737,23 +749,114 @@ test "newestCorporateAction: takes the max across dividends and splits" {
     defer a.free(dir_path);
 
     var store = cache.Store.init(io, a, dir_path);
+    const through = Date.fromYmd(2026, 8, 14);
 
     // Nothing cached at all.
-    try testing.expect(newestCorporateAction(a, &store, "SMPL") == null);
+    try testing.expect(newestCorporateAction(a, &store, "SMPL", through) == null);
 
     var divs = [_]Dividend{
         .{ .ex_date = Date.fromYmd(2026, 3, 20), .amount = 1.79 },
         .{ .ex_date = Date.fromYmd(2026, 6, 18), .amount = 1.90 },
     };
     store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
-    try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 6, 18)));
+    try testing.expect(newestCorporateAction(a, &store, "SMPL", through).?.eql(Date.fromYmd(2026, 6, 18)));
 
     // A later split must win over the later dividend.
     var splits = [_]Split{.{ .date = Date.fromYmd(2026, 7, 1), .numerator = 2, .denominator = 1 }};
     store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits });
-    try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 7, 1)));
+    try testing.expect(newestCorporateAction(a, &store, "SMPL", through).?.eql(Date.fromYmd(2026, 7, 1)));
 
     // Splits alone (different symbol) still resolve.
     store.write(Split, "SMPLB", splits[0..], .{ .seconds = cache.Ttl.splits });
-    try testing.expect(newestCorporateAction(a, &store, "SMPLB").?.eql(Date.fromYmd(2026, 7, 1)));
+    try testing.expect(newestCorporateAction(a, &store, "SMPLB", through).?.eql(Date.fromYmd(2026, 7, 1)));
+}
+
+test "newestCorporateAction: a forward-announced action does not mask an older unapplied one" {
+    // The regression. NKE announces roughly a quarter ahead, so its
+    // newest cached ex-date is essentially always in the future. Bounding
+    // the *verdict* on "is the newest action still in the future?" made
+    // that one announcement hide a distribution that had gone ex two
+    // months earlier, and NKE could never restate. The bound belongs on
+    // the selection, so the announcement is skipped and the older
+    // already-ex action is still found.
+    const io = testing.io;
+    const a = testing.allocator;
+    var tmp = testing.tmpDir(.{});
+    defer tmp.cleanup();
+    const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a);
+    defer a.free(dir_path);
+
+    var store = cache.Store.init(io, a, dir_path);
+    const last_bar = Date.fromYmd(2026, 8, 18);
+
+    // NKE's exact shape: applied through 2026-03-02, 2026-06-01 went ex
+    // and was never applied, 2026-09-01 declared but not yet ex.
+    var divs = [_]Dividend{
+        .{ .ex_date = Date.fromYmd(2026, 9, 1), .amount = 0.41 },
+        .{ .ex_date = Date.fromYmd(2026, 6, 1), .amount = 0.41 },
+        .{ .ex_date = Date.fromYmd(2026, 3, 2), .amount = 0.41 },
+    };
+    store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
+
+    // The future announcement is skipped; the newest already-ex wins.
+    const newest = newestCorporateAction(a, &store, "SMPL", last_bar);
+    try testing.expect(newest != null);
+    try testing.expect(newest.?.eql(Date.fromYmd(2026, 6, 1)));
+
+    // And the verdict is therefore "stale" for a legacy sentinel basis.
+    try testing.expect(adjustmentBasisStale(Date.epoch, newest));
+    // ...and for a basis covering 2026-03-02 but not 2026-06-01.
+    try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 4, 1), newest));
+    // ...but not once the basis has caught up.
+    try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 17), newest));
+}
+
+test "newestCorporateAction: a declared-but-not-effective split does not mask either" {
+    const io = testing.io;
+    const a = testing.allocator;
+    var tmp = testing.tmpDir(.{});
+    defer tmp.cleanup();
+    const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a);
+    defer a.free(dir_path);
+
+    var store = cache.Store.init(io, a, dir_path);
+    const last_bar = Date.fromYmd(2026, 8, 18);
+
+    var splits = [_]Split{
+        .{ .date = Date.fromYmd(2026, 10, 1), .numerator = 2, .denominator = 1 },
+        .{ .date = Date.fromYmd(2026, 5, 1), .numerator = 3, .denominator = 1 },
+    };
+    store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits });
+
+    const newest = newestCorporateAction(a, &store, "SMPL", last_bar);
+    try testing.expect(newest.?.eql(Date.fromYmd(2026, 5, 1)));
+    try testing.expect(adjustmentBasisStale(Date.epoch, newest));
+}
+
+test "newestCorporateAction: convergence - an only-future action is not actionable" {
+    // The property the bad bound was reaching for, preserved. A
+    // declared-but-not-yet-ex action is in no provider's adjustment
+    // series, so treating it as something to catch up to would refetch
+    // the full history on every pass, forever.
+    const io = testing.io;
+    const a = testing.allocator;
+    var tmp = testing.tmpDir(.{});
+    defer tmp.cleanup();
+    const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a);
+    defer a.free(dir_path);
+
+    var store = cache.Store.init(io, a, dir_path);
+
+    var divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 9, 17), .amount = 1.9 }};
+    store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
+
+    const last_bar = Date.fromYmd(2026, 8, 18);
+    try testing.expect(newestCorporateAction(a, &store, "SMPL", last_bar) == null);
+    try testing.expect(!adjustmentBasisStale(Date.epoch, newestCorporateAction(a, &store, "SMPL", last_bar)));
+
+    // Boundary: once it goes ex - `through` reaching the ex-date - it
+    // becomes actionable on that very bar.
+    const on_ex = Date.fromYmd(2026, 9, 17);
+    try testing.expect(newestCorporateAction(a, &store, "SMPL", on_ex).?.eql(on_ex));
+    try testing.expect(adjustmentBasisStale(Date.epoch, newestCorporateAction(a, &store, "SMPL", on_ex)));
 }

src/commands/diagnose.zig

@@ -254,8 +254,8 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
         // bars behind it - total returns then read low by roughly the
         // missed yield. Report it here because the symptom (a slightly
         // low 1Y total return) is otherwise invisible.
-        if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| {
+        if (freshness.newestCorporateAction(arena, &store, symbol, m.meta.last_date)) |newest_action| {
-            if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) {
+            if (freshness.adjustmentBasisStale(m.meta.adj_basis, newest_action)) {
                 try out.print(
                     "adj basis  {f} - STALE, {f} went ex behind it; total returns read low until restated\n",
                     .{ m.meta.adj_basis, newest_action },

src/service.zig

@@ -843,10 +843,21 @@ pub const DataService = struct {
         next.provider = provider;
         next.fail_count = 0;
 
+        // A provider change and a cleared backoff are independent facts,
+        // and conflating them hid the first one. This used to log only
+        // when clearing an armed backoff - but the case that mattered
+        // was a legacy cache carrying `provider = .yahoo` with no
+        // backoff at all, which is every symbol that had drifted off
+        // Tiingo before `tiingo_retry_after_s` existed. Twenty-one
+        // symbols converted back to Tiingo without a single line.
+        if (meta.provider != provider) {
+            log.info("{s}: candle provider {t} -> {t}", .{ symbol, meta.provider, provider });
+        }
+
         switch (coverage) {
             .covered => {
                 if (meta.tiingo_retry_after_s != 0) {
-                    log.info("{s}: provider converted {t} -> tiingo, clearing Tiingo backoff", .{ symbol, meta.provider });
+                    log.info("{s}: Tiingo serving again, clearing backoff", .{symbol});
                 }
                 next.tiingo_retry_after_s = 0;
             },
@@ -1147,9 +1158,9 @@ pub const DataService = struct {
                     // see a same-dated file whose historical adj_close is
                     // stale, so the basis has to be re-checked here.
                     const synced = if (s.readCandleMeta(symbol)) |sm| sm.meta else m;
-                    const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol);
+                    const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol, synced.last_date);
                     if (s.isCandleMetaFresh(symbol) and
-                        !freshness.adjustmentBasisStale(synced.adj_basis, synced.last_date, synced_action))
+                        !freshness.adjustmentBasisStale(synced.adj_basis, synced_action))
                     {
                         log.debug("{s}: candles synced from server and fresh", .{symbol});
                         if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
@@ -1180,8 +1191,7 @@ pub const DataService = struct {
                 // tax the hot portfolio-pricing path for nothing.
                 if (freshness.adjustmentBasisStale(
                     m.adj_basis,
-                    m.last_date,
+                    freshness.newestCorporateAction(self.allocator, &s, symbol, m.last_date),
-                    freshness.newestCorporateAction(self.allocator, &s, symbol),
                 )) {
                     log.info("{s}: restating full history (adj_basis {f} predates a corporate action)", .{ symbol, m.adj_basis });
                     if (self.refetchFullHistory(symbol, today, now_s, now_s < m.tiingo_retry_after_s)) |candles| {
@@ -4301,8 +4311,7 @@ test "getCandles offline never escalates a stale adjustment basis" {
     store.updateCandleMeta("SMPL", meta, 1); // expiry in the past => stale
     try std.testing.expect(freshness.adjustmentBasisStale(
         meta.adj_basis,
-        meta.last_date,
+        freshness.newestCorporateAction(allocator, &store, "SMPL", meta.last_date),
-        freshness.newestCorporateAction(allocator, &store, "SMPL"),
     ));
 
     svc.panic_on_network_attempt = true;