diff --git a/src/cache/freshness.zig b/src/cache/freshness.zig index 961cb85..7e1bbd3 100644 --- a/src/cache/freshness.zig +++ b/src/cache/freshness.zig @@ -30,11 +30,30 @@ //! //! Pure: dates and `now_s` in, findings out. No I/O, no cache reads, no //! fetches. The caller supplies the corpus. +//! +//! ## Adjustment-basis staleness +//! +//! A second, unrelated staleness question lives here too: +//! `adjustmentBasisStale` asks whether a symbol's cached `adj_close` +//! values predate a corporate action they should already reflect. It is +//! the same shape of problem - "this cache entry is quietly frozen" - +//! and it follows the same two-part split as the peer analysis above: +//! `newestCorporateAction` gathers from disk (like `collect`), and +//! `adjustmentBasisStale` decides (like `scan`, pure dates in, bool +//! out). +//! +//! It deliberately does not live in `store.zig`. The cache layer +//! serializes and reads; deciding that a distribution behind the basis +//! means the series must be refetched is domain policy, and +//! `store.zig`'s own `updateCandleMeta` doc records the same boundary +//! for market-clock knowledge ("owned by the caller"). const std = @import("std"); const Date = @import("../Date.zig"); const market = @import("../market.zig"); const cache = @import("store.zig"); +const Dividend = @import("../models/dividend.zig").Dividend; +const Split = @import("../models/split.zig").Split; /// One cached symbol, as the caller found it. pub const Entry = struct { @@ -603,3 +622,138 @@ test "collect-style exclusion: an excluded symbol produces no finding at all" { try testing.expectEqual(@as(usize, 0), without.orphans.len); try testing.expectEqual(@as(usize, 0), without.stale.len); } + +// ── Adjustment-basis staleness ─────────────────────────────── +// +// The candle cache is append-only. `getCandles` tops up from +// `last_date + 1`, and the newly-appended bars arrive with +// `adj_close == close` because nothing has gone ex after them yet - +// while every previously-cached bar keeps the adjustment basis it was +// originally fetched with. When the next distribution goes ex, the bars +// behind it should be marked down by its factor and nothing does it, so +// every total return spanning that ex-date reads low by roughly the +// missed yield. +// +// `CandleMeta.adj_basis` records how current a series' basis is. These +// two functions turn that into a verdict, split the same way as +// `collect` / `scan`: one gathers from disk, one decides. + +/// Newest corporate-action ex-date cached for a symbol, or null when no +/// dividends or splits are on disk. +/// +/// Splits count as much as dividends here, and arguably more: raw +/// `close` is not split-adjusted (hence `split.cumulativeSplitRatio`), +/// so an unapplied 2:1 split leaves older `adj_close` values wrong by +/// 50% rather than by a quarter's yield. +/// +/// Allocates only transiently - the returned `Date` borrows nothing. +pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, symbol: []const u8) ?Date { + var newest: ?Date = null; + + // `CacheResult` has no deinit - the caller owns `data`. Dividends + // carry owned strings, so they need `freeSlice`; splits are + // pure-numeric. + if (store.read(allocator, Dividend, symbol, null, .any)) |r| { + defer Dividend.freeSlice(allocator, r.data); + for (r.data) |d| { + if (newest == null or newest.?.lessThan(d.ex_date)) newest = d.ex_date; + } + } + if (store.read(allocator, Split, symbol, null, .any)) |r| { + defer allocator.free(r.data); + for (r.data) |sp| { + if (newest == null or newest.?.lessThan(sp.date)) newest = sp.date; + } + } + + return newest; +} + +/// Does a series' `adj_close` predate a corporate action it should +/// already reflect? +/// +/// - `adj_basis`: `CandleMeta.adj_basis` - the bar date through which +/// the cached `adj_close` values reflect corporate actions. +/// - `last_bar`: `CandleMeta.last_date` - the newest bar held. +/// - `newest_action`: from `newestCorporateAction`, or null for a +/// symbol with no cached dividends or splits. +/// +/// The `last_bar` bound is what keeps this convergent. A distribution +/// announced with a future ex-date is not yet reflected in *any* +/// provider's adjustment series, so treating it as stale would refetch +/// on every pass and never settle. +/// +/// Pure: three dates in, bool out. +pub fn adjustmentBasisStale(adj_basis: Date, last_bar: Date, newest_action: ?Date) bool { + const newest = newest_action orelse return false; + if (last_bar.lessThan(newest)) return false; + return adj_basis.lessThan(newest); +} + +test "adjustmentBasisStale: no corporate action means nothing to restate" { + // A non-payer has no ex-date to exceed the basis, so it must never + // escalate - not even with the epoch sentinel a legacy cache parses to. + try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), Date.fromYmd(2026, 8, 14), null)); + try testing.expect(!adjustmentBasisStale(Date.epoch, Date.fromYmd(2026, 8, 14), null)); +} + +test "adjustmentBasisStale: action behind the basis is stale" { + const last_bar = Date.fromYmd(2026, 8, 14); + const ex = Date.fromYmd(2026, 6, 18); + + // Basis at the newest bar already covers the ex-date. + try testing.expect(!adjustmentBasisStale(last_bar, last_bar, ex)); + // Basis exactly at the ex-date covers it too (not `lessThan`). + try testing.expect(!adjustmentBasisStale(ex, last_bar, ex)); + // Basis behind the ex-date: the bars in between were never marked down. + try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), last_bar, ex)); + // The legacy-cache case: sentinel basis on a dividend payer. This is + // the shape every pre-adj_basis cache lands in. + try testing.expect(adjustmentBasisStale(Date.epoch, last_bar, ex)); +} + +test "adjustmentBasisStale: a not-yet-ex action is not stale" { + // Ex-date beyond the newest bar we hold. No provider has applied it + // either, so escalating would never converge. + try testing.expect(!adjustmentBasisStale( + Date.fromYmd(2026, 5, 1), + Date.fromYmd(2026, 8, 14), + Date.fromYmd(2026, 9, 17), + )); + // Boundary: an ex-date exactly on the newest bar IS covered. + try testing.expect(adjustmentBasisStale( + Date.fromYmd(2026, 5, 1), + Date.fromYmd(2026, 8, 14), + Date.fromYmd(2026, 8, 14), + )); +} + +test "newestCorporateAction: takes the max across dividends and splits" { + const io = testing.io; + const a = testing.allocator; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir_path); + + var store = cache.Store.init(io, a, dir_path); + + // Nothing cached at all. + try testing.expect(newestCorporateAction(a, &store, "SMPL") == null); + + var divs = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2026, 3, 20), .amount = 1.79 }, + .{ .ex_date = Date.fromYmd(2026, 6, 18), .amount = 1.90 }, + }; + store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends }); + try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 6, 18))); + + // A later split must win over the later dividend. + var splits = [_]Split{.{ .date = Date.fromYmd(2026, 7, 1), .numerator = 2, .denominator = 1 }}; + store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits }); + try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 7, 1))); + + // Splits alone (different symbol) still resolve. + store.write(Split, "SMPLB", splits[0..], .{ .seconds = cache.Ttl.splits }); + try testing.expect(newestCorporateAction(a, &store, "SMPLB").?.eql(Date.fromYmd(2026, 7, 1))); +} diff --git a/src/cache/store.zig b/src/cache/store.zig index 66b6fa7..22845ee 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -906,8 +906,15 @@ pub const Store = struct { /// (the caller computes the market-aware freshness boundary; see /// `market.nextCandleExpiry`). /// - /// `attrs` supplies the provider-state fields; `last_close` and - /// `last_date` are derived from the newest candle written. + /// `attrs` supplies the provider-state fields; `last_close`, + /// `last_date` and `adj_basis` are derived from the newest candle + /// written. + /// + /// Deriving `adj_basis` here is the whole point of routing full + /// refetches through this function: replacing the file means every + /// bar carries the provider's current adjustment basis, which by + /// construction reflects every corporate action up to the newest + /// bar. `appendCandles` deliberately cannot do this. pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, attrs: CandleMetaAttrs, expires_at_s: i64) void { if (serializeCandles(self.allocator, candles, .{})) |srf_data| { defer self.allocator.free(srf_data); @@ -926,6 +933,7 @@ pub const Store = struct { .provider = attrs.provider, .fail_count = attrs.fail_count, .tiingo_retry_after_s = attrs.tiingo_retry_after_s, + .adj_basis = last.date, }, expires_at_s); } } @@ -940,7 +948,10 @@ pub const Store = struct { /// Callers are expected to pass the symbol's *existing* meta so /// that fields describing the series as a whole survive an append /// unchanged - appending bars does not re-derive anything about - /// the rows already on disk. + /// the rows already on disk. `adj_basis` in particular MUST NOT + /// advance here: the appended bars do not restate the older rows' + /// `adj_close`, so claiming a newer basis would mask exactly the + /// staleness that field exists to detect. pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, meta: CandleMeta, expires_at_s: i64) void { if (new_candles.len == 0) return; @@ -1426,6 +1437,37 @@ pub const Store = struct { /// Defaulted so legacy caches (which lack the field) parse /// cleanly and simply behave as "no backoff". tiingo_retry_after_s: i64 = 0, + /// The bar date through which this series' `adj_close` values + /// reflect corporate actions - i.e. the newest bar present at + /// the last *full* fetch. + /// + /// Providers compute `adj_close` by scaling raw `close` by the + /// product of the adjustment factors for every distribution + /// *after* that bar. So the whole series' adjustment basis is + /// only as current as the fetch that produced it: a full fetch + /// as of date D reflects every ex-date <= D, and nothing later. + /// + /// Appends never advance this. `getCandles` tops the cache up + /// incrementally, and the newly-appended bars arrive with + /// `adj_close == close` (nothing has gone ex after them yet) + /// while every previously-cached bar keeps its original basis. + /// When a distribution later goes ex, the bars behind it should + /// be marked down and nothing does it - so total returns read + /// low by roughly the missed yield. Comparing this field + /// against the newest known dividend/split ex-date is how + /// `getCandles` detects that and escalates to a full refetch. + /// + /// `Date.epoch` (the default) means "unknown - assume stale". + /// Any real ex-date is later than it, so a dividend payer + /// escalates on its first stale pass; a symbol with no + /// corporate actions has no ex-date to exceed it and never + /// escalates. Defaulted rather than required precisely so + /// legacy caches keep parsing: making it required would route + /// every cached symbol through the cold-start path, which is + /// Tiingo-only and writes a negative-cache marker *over* the + /// candle file on a 404 - destroying history for any symbol + /// Tiingo does not carry. + adj_basis: Date = Date.epoch, }; /// The subset of `CandleMeta` that describes provider state rather @@ -3760,6 +3802,113 @@ test "appendCandles preserves caller-supplied provider state" { try std.testing.expectApproxEqAbs(@as(f64, 2), after.meta.last_close, 0.001); } +// ── adj_basis: adjustment-basis staleness ──────────────────── +// +// The cache is append-only. A distribution that goes ex after the last +// full fetch never marks down the bars behind it, so total returns read +// low by roughly the missed yield. `adj_basis` records how current the +// series' adjustment basis is; these tests pin the detection. + +/// Build a one-bar-per-day series ending at `last`, for basis tests. +fn basisTestCandles(buf: []Candle, first: Date, count: usize) []Candle { + for (0..count) |i| { + const d = first.addDays(@intCast(i)); + buf[i] = .{ .date = d, .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 }; + } + return buf[0..count]; +} + +test "cacheCandles stamps adj_basis at the newest bar" { + // A full replace means every row carries the provider's current + // adjustment basis, which by construction covers every corporate + // action up to the newest bar. + const io = std.testing.io; + const allocator = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + var buf: [5]Candle = undefined; + const candles = basisTestCandles(&buf, Date.fromYmd(2026, 8, 10), 5); + s.cacheCandles("SMPL", candles, .{ .provider = .tiingo }, 9_999_999_999); + + const meta = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta; + try std.testing.expect(meta.adj_basis.eql(Date.fromYmd(2026, 8, 14))); + try std.testing.expect(meta.last_date.eql(Date.fromYmd(2026, 8, 14))); +} + +test "appendCandles does not advance adj_basis" { + // The core invariant. Appended bars arrive with adj_close == close + // (nothing has gone ex after them yet) and do NOT restate the rows + // already on disk, so claiming a newer basis would mask exactly the + // staleness this field exists to detect. + const io = std.testing.io; + const allocator = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + var buf: [3]Candle = undefined; + s.cacheCandles("SMPL", basisTestCandles(&buf, Date.fromYmd(2026, 6, 1), 3), .{ .provider = .tiingo }, 9_999_999_999); + const before = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta; + try std.testing.expect(before.adj_basis.eql(Date.fromYmd(2026, 6, 3))); + + var more: [3]Candle = undefined; + s.appendCandles("SMPL", basisTestCandles(&more, Date.fromYmd(2026, 8, 12), 3), before, 9_999_999_999); + + const after = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta; + // last_date advanced... + try std.testing.expect(after.last_date.eql(Date.fromYmd(2026, 8, 14))); + // ...but the basis is pinned to the last full fetch. + try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 3))); +} + +test "legacy candles_meta without adj_basis parses to the epoch sentinel" { + // Same migration story as tiingo_retry_after_s, and the stakes are + // higher: making this field required would route every cached symbol + // through the cold-start path, which on a 404 writes a negative + // marker OVER candles_daily.srf and destroys the history. + const allocator = std.testing.allocator; + const legacy = + \\#!srfv1 + \\#!expires=1787000100 + \\#!created=1786748010 + \\last_close:num:776.34,last_date::2026-08-14,provider::yahoo + \\ + ; + const parsed = try Store.deserializeCandleMeta(allocator, legacy); + try std.testing.expect(parsed.adj_basis.eql(Date.epoch)); + // A sentinel basis on a dividend payer is exactly the "needs + // restatement" signal, so it must not be mistaken for "current". + try std.testing.expect(parsed.adj_basis.lessThan(Date.fromYmd(2026, 6, 18))); +} + +test "adj_basis is elided when unset and round-trips when set" { + const allocator = std.testing.allocator; + + const unset = Store.CandleMeta{ + .last_close = 100.0, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .tiingo, + }; + const bare = try Store.serializeCandleMeta(std.testing.io, allocator, unset, .{ .expires = 1 }); + defer allocator.free(bare); + try std.testing.expect(std.mem.indexOf(u8, bare, "adj_basis") == null); + + var set = unset; + set.adj_basis = Date.fromYmd(2026, 8, 14); + const full = try Store.serializeCandleMeta(std.testing.io, allocator, set, .{ .expires = 1 }); + defer allocator.free(full); + try std.testing.expect(std.mem.indexOf(u8, full, "adj_basis::2026-08-14") != null); + + const parsed = try Store.deserializeCandleMeta(allocator, full); + try std.testing.expect(parsed.adj_basis.eql(Date.fromYmd(2026, 8, 14))); +} + // ── writeRaw / appendRaw atomicity ─────────────────────────── // // A concurrent reader hitting a cache file mid-write must never see a diff --git a/src/commands/diagnose.zig b/src/commands/diagnose.zig index 5bef635..de97471 100644 --- a/src/commands/diagnose.zig +++ b/src/commands/diagnose.zig @@ -248,6 +248,27 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void { try out.print(", Tiingo backoff lapsed (retries next refresh)", .{}); } try out.print("\n", .{}); + + // Adjustment basis. The cache is append-only, so a distribution + // that goes ex after the last full fetch never marks down the + // bars behind it - total returns then read low by roughly the + // missed yield. Report it here because the symptom (a slightly + // low 1Y total return) is otherwise invisible. + if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| { + if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) { + try out.print( + "adj basis {f} - STALE, {f} went ex behind it; total returns read low until restated\n", + .{ m.meta.adj_basis, newest_action }, + ); + } else { + try out.print( + "adj basis {f} - current through the newest corporate action ({f})\n", + .{ m.meta.adj_basis, newest_action }, + ); + } + } else { + try out.print("adj basis {f} - no dividends or splits cached, nothing to restate\n", .{m.meta.adj_basis}); + } } else { try out.print("local nothing cached\n", .{}); } diff --git a/src/service.zig b/src/service.zig index 2d401fc..04abe3d 100644 --- a/src/service.zig +++ b/src/service.zig @@ -23,6 +23,7 @@ const Holding = @import("models/etf_profile.zig").Holding; const SectorWeight = @import("models/etf_profile.zig").SectorWeight; const Config = @import("Config.zig"); const cache = @import("cache/store.zig"); +const freshness = @import("cache/freshness.zig"); const srf = @import("srf"); const analysis = @import("analytics/analysis.zig"); const transaction_log = @import("models/transaction_log.zig"); @@ -677,6 +678,112 @@ pub const DataService = struct { return triple; } + /// Fixed start date for any full-history candle fetch. See + /// `populateAllFromTiingo`'s doc comment for the rationale. + const full_history_start: Date = Date.fromYmd(2000, 1, 1); + + /// Replace a symbol's entire candle series, restating `adj_close` + /// from whichever provider will serve it. + /// + /// This is the only way a cached series' adjustment basis ever + /// advances - see `CandleMeta.adj_basis`. Incremental appends + /// cannot restate the rows already on disk, so once a distribution + /// goes ex behind them the whole series reads low until something + /// rewrites it. That something is this function. + /// + /// Tiingo first (it returns dividends and splits in the same + /// response, and its `adj_close` is the one the analytics layer is + /// written against), falling back to a full-range Yahoo fetch when + /// Tiingo does not carry the symbol. + /// + /// **Never writes a negative-cache entry.** Callers reach this + /// holding a working series; `writeNegative` would overwrite + /// `candles_daily.srf` with a marker and destroy that history. A + /// restatement that cannot be completed is a "try again later", not + /// a verdict about the symbol. Callers should treat any error as + /// "keep the existing series and carry on". + /// + /// Returns `error.NotFound` only when *every* provider says it does + /// not carry the symbol - that is the one outcome a cold-start + /// caller may legitimately turn into a negative-cache entry. Any + /// other failure surfaces as `FetchFailed` / `TransientError` / + /// `AuthError` so a network blip can never be mistaken for + /// "this symbol does not exist". + fn refetchFullHistory( + self: *DataService, + symbol: []const u8, + today: Date, + now_s: i64, + prefer_yahoo: bool, + ) (DataError || error{NotFound})![]Candle { + self.assertNetworkAllowed("getCandles refetchFullHistory"); + const kind = market.classify(symbol); + var s = self.store(); + + // Tracks whether each provider affirmatively said "no such + // symbol", as opposed to failing for some other reason. Only + // unanimous 404s justify the caller writing a negative entry. + var tiingo_not_found = false; + + if (!prefer_yahoo) { + if (self.populateAllFromTiingo(symbol)) |triple| { + defer Dividend.freeSlice(self.allocator, triple.dividends); + defer self.allocator.free(triple.splits); + if (triple.candles.len > 0) return triple.candles; + // An empty full-history response is not a restatement; + // fall through rather than caching emptiness. + self.allocator.free(triple.candles); + log.warn("{s}: Tiingo full history returned no bars, trying Yahoo", .{symbol}); + } else |err| { + // Transient failures must not silently degrade to a + // second provider - the caller needs to know to retry. + if (err == error.RateLimited or isTransientError(err)) return DataError.TransientError; + if (err == error.Unauthorized) { + log.err("{s}: Tiingo auth failed during restatement - check TIINGO_API_KEY", .{symbol}); + return DataError.AuthError; + } + tiingo_not_found = isPermanentProviderFailure(err); + log.info("{s}: Tiingo cannot serve full history ({s}), trying Yahoo", .{ symbol, @errorName(err) }); + } + } + + // Yahoo fallback. `fetchCandles` takes an arbitrary range, so a + // full-history request is just a wide one. Yahoo carries no + // dividend/split endpoint here; those caches are Polygon-primary + // and merged separately, so leaving them untouched is correct. + if (self.getProvider(Yahoo)) |yh| { + if (yh.fetchCandles(self.allocator, symbol, full_history_start, today)) |candles| { + if (candles.len > 0) { + s.cacheCandles(symbol, candles, .{ + .provider = .yahoo, + // Preserve the Tiingo verdict this call just + // learned, so the next pass doesn't re-probe. + .tiingo_retry_after_s = if (tiingo_not_found) + cache.computeExpires( + now_s, + .{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct }, + symbol, + ) + else + 0, + }, expiryAfterFetch(now_s, kind, candles)); + log.info("{s}: full history restated from Yahoo ({d} bars)", .{ symbol, candles.len }); + return candles; + } + self.allocator.free(candles); + log.warn("{s}: Yahoo full history returned no bars", .{symbol}); + } else |err| { + log.warn("{s}: Yahoo full history failed: {s}", .{ symbol, @errorName(err) }); + // Both providers affirmatively disclaim the symbol. + if (tiingo_not_found and isPermanentProviderFailure(err)) return error.NotFound; + } + } else |_| { + log.warn("{s}: Yahoo provider not available for full history", .{symbol}); + } + + return DataError.FetchFailed; + } + /// Invalidate cached data for a symbol so the next get* call forces a fresh fetch. pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void { var s = self.store(); @@ -1032,7 +1139,17 @@ pub const DataService = struct { // (Force-refresh skips server sync too: the user explicitly // asked for fresh provider data.) if (!opts.force_refresh and self.syncCandlesFromServer(symbol)) { - if (s.isCandleMetaFresh(symbol)) { + // Re-read meta: the sync wrote the server's bytes + // verbatim, so its view of both freshness AND + // adjustment basis is now ours. `serverBarRegression` + // only guards `last_date` going backwards - it cannot + // see a same-dated file whose historical adj_close is + // stale, so the basis has to be re-checked here. + const synced = if (s.readCandleMeta(symbol)) |sm| sm.meta else m; + const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol); + if (s.isCandleMetaFresh(symbol) and + !freshness.adjustmentBasisStale(synced.adj_basis, synced.last_date, synced_action)) + { log.debug("{s}: candles synced from server and fresh", .{symbol}); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; @@ -1047,6 +1164,36 @@ pub const DataService = struct { // this stale path (next post-close / NAV-availability time). const expires = market.nextCandleExpiry(now_s, kind); + // A corporate action has gone ex since the basis that + // produced this series, so the cached `adj_close` values + // behind it were never marked down. Appending cannot fix + // that - only replacing the file can. Do this BEFORE the + // incremental fetch so a symbol that needs both a top-up + // and a restatement costs one full fetch, not an append + // followed by a second pass. + // + // Deliberately not checked on the fresh-cache path above: + // the candle TTL lapses at least once per trading day, so + // detection is at most a day behind, and paying a + // dividend/split cache read on every getCandles call would + // tax the hot portfolio-pricing path for nothing. + if (freshness.adjustmentBasisStale( + m.adj_basis, + m.last_date, + freshness.newestCorporateAction(self.allocator, &s, symbol), + )) { + log.info("{s}: restating full history (adj_basis {f} predates a corporate action)", .{ symbol, m.adj_basis }); + if (self.refetchFullHistory(symbol, today, now_s, now_s < m.tiingo_retry_after_s)) |candles| { + return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; + } else |err| { + // Restatement is best-effort. The existing series + // is untouched and still usable, just understated + // by the missed adjustment - fall through to the + // normal top-up and retry on the next stale pass. + log.warn("{s}: full-history restatement failed ({s}); adjustment basis remains stale, falling back to incremental append", .{ symbol, @errorName(err) }); + } + } + // Only skip the fetch when we already hold the latest // *available* bar (weekend/holiday/pre-close gap, or // caught up): just bump the TTL. Gating on the @@ -1140,19 +1287,27 @@ pub const DataService = struct { log.debug("{s}: candles synced from server but stale, falling through to full fetch", .{symbol}); } - // No usable cache - full fetch via the orchestrated Tiingo - // helper, which writes candles + dividends + splits caches in - // one shot from a single HTTP response. The fixed start date - // (see `populateAllFromTiingo`) is 2000-01-01, deep enough to - // cover a 10Y trailing-return window even when `--as-of` - // back-dates the reference into 2014-era imported portfolio - // history, plus a buffer for older corporate actions like - // SPYM's 2017-10-16 split. + // No usable cache - full fetch. Tiingo first (it returns + // candles + dividends + splits from one response), falling back + // to a full-range Yahoo fetch when Tiingo does not carry the + // symbol. The fixed start date (see `populateAllFromTiingo`) is + // 2000-01-01, deep enough to cover a 10Y trailing-return window + // even when `--as-of` back-dates the reference into 2014-era + // imported portfolio history, plus a buffer for older corporate + // actions like SPYM's 2017-10-16 split. + // + // The Yahoo fallback matters here beyond convenience: this + // branch used to be Tiingo-only, so a symbol Tiingo does not + // carry could not be cold-started at all - it 404'd, wrote a + // negative-cache marker *over* candles_daily.srf, and stayed + // unavailable until `cache clear`. Any symbol that reached this + // branch with a populated cache would have had its history + // destroyed. log.debug("{s}: fetching full candle history from provider", .{symbol}); - self.assertNetworkAllowed("getCandles full populateAllFromTiingo"); - const triple = self.populateAllFromTiingo(symbol) catch |err| { - if (err == error.RateLimited or err == error.ServerError or err == error.RequestFailed) { + const prior_backoff: i64 = if (meta_result) |mr| mr.meta.tiingo_retry_after_s else 0; + const candles = self.refetchFullHistory(symbol, today, now_s, now_s < prior_backoff) catch |err| { + if (err == DataError.TransientError) { // Transient: increment fail_count on existing meta so // we know to back off if this keeps happening. if (meta_result) |mr| { @@ -1162,24 +1317,20 @@ pub const DataService = struct { } return DataError.TransientError; } - // Only a genuine NotFound means "this symbol has no candle - // data on Tiingo" (the sole historical-candle provider since - // the 2026-05 audit) and earns a sticky negative-cache entry. - // Unauthorized / InvalidResponse / PaymentRequired and the - // like are not permanent facts about the symbol (transient - // auth misconfig, a malformed response) - fail this call but - // stay retryable, matching fetchCached's policy. This matters - // now that the candle negative cache is actually honored: a - // bad negative would otherwise stick until --refresh. - if (isPermanentProviderFailure(err)) s.writeNegative(symbol, .candles_daily); + // Only a unanimous NotFound - every provider affirmatively + // disclaims the symbol - earns a sticky negative-cache + // entry. Auth trouble, a malformed response, or a Yahoo + // network blip are not permanent facts about the symbol, so + // they fail this call but stay retryable. This matters + // because the negative marker replaces the candle file: a + // bad negative both suppresses retries until `--refresh` + // and throws away whatever history was cached. + if (err == error.NotFound) s.writeNegative(symbol, .candles_daily); + if (err == DataError.AuthError) return DataError.AuthError; return DataError.FetchFailed; }; - // populateAllFromTiingo writes all three caches itself; we - // free the slices we don't return. - defer Dividend.freeSlice(self.allocator, triple.dividends); - defer self.allocator.free(triple.splits); - return .{ .data = triple.candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; + return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } /// Fetch dividend history for a symbol. @@ -4079,8 +4230,58 @@ test "loadAllPrices offline mode skips network and returns cached" { try std.testing.expectEqual(@as(usize, 1), result.failed_count); } -// ── Tiingo coverage bookkeeping ────────────────────────────── -// +// ── adjustment-basis restatement ───────────────────────────── + +test "getCandles offline never escalates a stale adjustment basis" { + // Restatement does network I/O, so `skip_network` must return the + // cached series untouched rather than escalating. Pinned with + // `panic_on_network_attempt`, which fires inside + // `refetchFullHistory`'s `assertNetworkAllowed` if this regresses. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + const config = Config{ .cache_dir = dir_path }; + var svc = DataService.init(io, allocator, config); + defer svc.deinit(); + + var store = svc.store(); + var candles = [_]Candle{ + .{ .date = Date.fromYmd(2026, 8, 12), .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 }, + .{ .date = Date.fromYmd(2026, 8, 13), .open = 11, .high = 11, .low = 11, .close = 11, .adj_close = 11, .volume = 1 }, + }; + store.cacheCandles("SMPL", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); + + // Roll the basis back behind a cached ex-date so the series is + // unambiguously due for restatement. + var divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 7, 1), .amount = 1.0 }}; + store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends }); + var meta = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta; + meta.adj_basis = Date.fromYmd(2026, 6, 1); + store.updateCandleMeta("SMPL", meta, 1); // expiry in the past => stale + try std.testing.expect(freshness.adjustmentBasisStale( + meta.adj_basis, + meta.last_date, + freshness.newestCorporateAction(allocator, &store, "SMPL"), + )); + + svc.panic_on_network_attempt = true; + const result = try svc.getCandles("SMPL", .{ .skip_network = true }); + defer result.deinit(); + + // Served from cache, series intact, basis untouched. + try std.testing.expectEqual(@as(usize, 2), result.data.len); + const after = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta; + try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 1))); + // And critically, no negative-cache marker was written over the + // candle file. + try std.testing.expect(!store.isNegative("SMPL", .candles_daily)); +} + +// ── Tiingo coverage bookkeeping ──────────────────────────────// // Regression suite for the one-way drift to Yahoo. The old code // routed on `CandleMeta.provider == .yahoo`, so any non-transient // Tiingo failure latched permanently: Yahoo was tried first,