lobo/zfin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

restate adj_close when a corporate action goes ex

Browse source
This commit is contained in:
Emil Lerch 2026-08-17 18:26:36 -07:00
parent c4680dbe45
commit 7163610b8e
Signed by: lobo
GPG key ID: A7B62D657EF764F8
4 changed files with 557 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
src/cache/freshness.zig vendored
View file

@ -30,11 +30,30 @@
//!
//! Pure: dates and `now_s` in, findings out. No I/O, no cache reads, no
//! fetches. The caller supplies the corpus.
//!
//! ## Adjustment-basis staleness
//!
//! A second, unrelated staleness question lives here too:
//! `adjustmentBasisStale` asks whether a symbol's cached `adj_close`
//! values predate a corporate action they should already reflect. It is
//! the same shape of problem - "this cache entry is quietly frozen" -
//! and it follows the same two-part split as the peer analysis above:
//! `newestCorporateAction` gathers from disk (like `collect`), and
//! `adjustmentBasisStale` decides (like `scan`, pure dates in, bool
//! out).
//!
//! It deliberately does not live in `store.zig`. The cache layer
//! serializes and reads; deciding that a distribution behind the basis
//! means the series must be refetched is domain policy, and
//! `store.zig`'s own `updateCandleMeta` doc records the same boundary
//! for market-clock knowledge ("owned by the caller").
const std = @import("std");
const Date = @import("../Date.zig");
const market = @import("../market.zig");
const cache = @import("store.zig");
const Dividend = @import("../models/dividend.zig").Dividend;
const Split = @import("../models/split.zig").Split;
/// One cached symbol, as the caller found it.
pub const Entry = struct {
@ -603,3 +622,138 @@ test "collect-style exclusion: an excluded symbol produces no finding at all" {
try testing.expectEqual(@as(usize, 0), without.orphans.len);
try testing.expectEqual(@as(usize, 0), without.stale.len);
}
// ── Adjustment-basis staleness ───────────────────────────────
//
// The candle cache is append-only. `getCandles` tops up from
// `last_date + 1`, and the newly-appended bars arrive with
// `adj_close == close` because nothing has gone ex after them yet -
// while every previously-cached bar keeps the adjustment basis it was
// originally fetched with. When the next distribution goes ex, the bars
// behind it should be marked down by its factor and nothing does it, so
// every total return spanning that ex-date reads low by roughly the
// missed yield.
//
// `CandleMeta.adj_basis` records how current a series' basis is. These
// two functions turn that into a verdict, split the same way as
// `collect` / `scan`: one gathers from disk, one decides.
/// Newest corporate-action ex-date cached for a symbol, or null when no
/// dividends or splits are on disk.
///
/// Splits count as much as dividends here, and arguably more: raw
/// `close` is not split-adjusted (hence `split.cumulativeSplitRatio`),
/// so an unapplied 2:1 split leaves older `adj_close` values wrong by
/// 50% rather than by a quarter's yield.
///
/// Allocates only transiently - the returned `Date` borrows nothing.
pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, symbol: []const u8) ?Date {
var newest: ?Date = null;
// `CacheResult` has no deinit - the caller owns `data`. Dividends
// carry owned strings, so they need `freeSlice`; splits are
// pure-numeric.
if (store.read(allocator, Dividend, symbol, null, .any)) |r| {
defer Dividend.freeSlice(allocator, r.data);
for (r.data) |d| {
if (newest == null or newest.?.lessThan(d.ex_date)) newest = d.ex_date;
}
}
if (store.read(allocator, Split, symbol, null, .any)) |r| {
defer allocator.free(r.data);
for (r.data) |sp| {
if (newest == null or newest.?.lessThan(sp.date)) newest = sp.date;
}
}
return newest;
}
/// Does a series' `adj_close` predate a corporate action it should
/// already reflect?
///
/// - `adj_basis`: `CandleMeta.adj_basis` - the bar date through which
/// the cached `adj_close` values reflect corporate actions.
/// - `last_bar`: `CandleMeta.last_date` - the newest bar held.
/// - `newest_action`: from `newestCorporateAction`, or null for a
/// symbol with no cached dividends or splits.
///
/// The `last_bar` bound is what keeps this convergent. A distribution
/// announced with a future ex-date is not yet reflected in *any*
/// provider's adjustment series, so treating it as stale would refetch
/// on every pass and never settle.
///
/// Pure: three dates in, bool out.
pub fn adjustmentBasisStale(adj_basis: Date, last_bar: Date, newest_action: ?Date) bool {
const newest = newest_action orelse return false;
if (last_bar.lessThan(newest)) return false;
return adj_basis.lessThan(newest);
}
test "adjustmentBasisStale: no corporate action means nothing to restate" {
// A non-payer has no ex-date to exceed the basis, so it must never
// escalate - not even with the epoch sentinel a legacy cache parses to.
try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), Date.fromYmd(2026, 8, 14), null));
try testing.expect(!adjustmentBasisStale(Date.epoch, Date.fromYmd(2026, 8, 14), null));
}
test "adjustmentBasisStale: action behind the basis is stale" {
const last_bar = Date.fromYmd(2026, 8, 14);
const ex = Date.fromYmd(2026, 6, 18);
// Basis at the newest bar already covers the ex-date.
try testing.expect(!adjustmentBasisStale(last_bar, last_bar, ex));
// Basis exactly at the ex-date covers it too (not `lessThan`).
try testing.expect(!adjustmentBasisStale(ex, last_bar, ex));
// Basis behind the ex-date: the bars in between were never marked down.
try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), last_bar, ex));
// The legacy-cache case: sentinel basis on a dividend payer. This is
// the shape every pre-adj_basis cache lands in.
try testing.expect(adjustmentBasisStale(Date.epoch, last_bar, ex));
}
test "adjustmentBasisStale: a not-yet-ex action is not stale" {
// Ex-date beyond the newest bar we hold. No provider has applied it
// either, so escalating would never converge.
try testing.expect(!adjustmentBasisStale(
Date.fromYmd(2026, 5, 1),
Date.fromYmd(2026, 8, 14),
Date.fromYmd(2026, 9, 17),
));
// Boundary: an ex-date exactly on the newest bar IS covered.
try testing.expect(adjustmentBasisStale(
Date.fromYmd(2026, 5, 1),
Date.fromYmd(2026, 8, 14),
Date.fromYmd(2026, 8, 14),
));
}
test "newestCorporateAction: takes the max across dividends and splits" {
const io = testing.io;
const a = testing.allocator;
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a);
defer a.free(dir_path);
var store = cache.Store.init(io, a, dir_path);
// Nothing cached at all.
try testing.expect(newestCorporateAction(a, &store, "SMPL") == null);
var divs = [_]Dividend{
.{ .ex_date = Date.fromYmd(2026, 3, 20), .amount = 1.79 },
.{ .ex_date = Date.fromYmd(2026, 6, 18), .amount = 1.90 },
};
store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 6, 18)));
// A later split must win over the later dividend.
var splits = [_]Split{.{ .date = Date.fromYmd(2026, 7, 1), .numerator = 2, .denominator = 1 }};
store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits });
try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 7, 1)));
// Splits alone (different symbol) still resolve.
store.write(Split, "SMPLB", splits[0..], .{ .seconds = cache.Ttl.splits });
try testing.expect(newestCorporateAction(a, &store, "SMPLB").?.eql(Date.fromYmd(2026, 7, 1)));
}

155
src/cache/store.zig vendored
View file

@ -906,8 +906,15 @@ pub const Store = struct {
/// (the caller computes the market-aware freshness boundary; see
/// `market.nextCandleExpiry`).
///
/// `attrs` supplies the provider-state fields; `last_close` and
/// `last_date` are derived from the newest candle written.
/// `attrs` supplies the provider-state fields; `last_close`,
/// `last_date` and `adj_basis` are derived from the newest candle
/// written.
///
/// Deriving `adj_basis` here is the whole point of routing full
/// refetches through this function: replacing the file means every
/// bar carries the provider's current adjustment basis, which by
/// construction reflects every corporate action up to the newest
/// bar. `appendCandles` deliberately cannot do this.
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, attrs: CandleMetaAttrs, expires_at_s: i64) void {
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
defer self.allocator.free(srf_data);
@ -926,6 +933,7 @@ pub const Store = struct {
.provider = attrs.provider,
.fail_count = attrs.fail_count,
.tiingo_retry_after_s = attrs.tiingo_retry_after_s,
.adj_basis = last.date,
}, expires_at_s);
}
}
@ -940,7 +948,10 @@ pub const Store = struct {
/// Callers are expected to pass the symbol's *existing* meta so
/// that fields describing the series as a whole survive an append
/// unchanged - appending bars does not re-derive anything about
/// the rows already on disk.
/// the rows already on disk. `adj_basis` in particular MUST NOT
/// advance here: the appended bars do not restate the older rows'
/// `adj_close`, so claiming a newer basis would mask exactly the
/// staleness that field exists to detect.
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, meta: CandleMeta, expires_at_s: i64) void {
if (new_candles.len == 0) return;
@ -1426,6 +1437,37 @@ pub const Store = struct {
/// Defaulted so legacy caches (which lack the field) parse
/// cleanly and simply behave as "no backoff".
tiingo_retry_after_s: i64 = 0,
/// The bar date through which this series' `adj_close` values
/// reflect corporate actions - i.e. the newest bar present at
/// the last *full* fetch.
///
/// Providers compute `adj_close` by scaling raw `close` by the
/// product of the adjustment factors for every distribution
/// *after* that bar. So the whole series' adjustment basis is
/// only as current as the fetch that produced it: a full fetch
/// as of date D reflects every ex-date <= D, and nothing later.
///
/// Appends never advance this. `getCandles` tops the cache up
/// incrementally, and the newly-appended bars arrive with
/// `adj_close == close` (nothing has gone ex after them yet)
/// while every previously-cached bar keeps its original basis.
/// When a distribution later goes ex, the bars behind it should
/// be marked down and nothing does it - so total returns read
/// low by roughly the missed yield. Comparing this field
/// against the newest known dividend/split ex-date is how
/// `getCandles` detects that and escalates to a full refetch.
///
/// `Date.epoch` (the default) means "unknown - assume stale".
/// Any real ex-date is later than it, so a dividend payer
/// escalates on its first stale pass; a symbol with no
/// corporate actions has no ex-date to exceed it and never
/// escalates. Defaulted rather than required precisely so
/// legacy caches keep parsing: making it required would route
/// every cached symbol through the cold-start path, which is
/// Tiingo-only and writes a negative-cache marker *over* the
/// candle file on a 404 - destroying history for any symbol
/// Tiingo does not carry.
adj_basis: Date = Date.epoch,
};
/// The subset of `CandleMeta` that describes provider state rather
@ -3760,6 +3802,113 @@ test "appendCandles preserves caller-supplied provider state" {
try std.testing.expectApproxEqAbs(@as(f64, 2), after.meta.last_close, 0.001);
}
// ── adj_basis: adjustment-basis staleness ────────────────────
//
// The cache is append-only. A distribution that goes ex after the last
// full fetch never marks down the bars behind it, so total returns read
// low by roughly the missed yield. `adj_basis` records how current the
// series' adjustment basis is; these tests pin the detection.
/// Build a one-bar-per-day series ending at `last`, for basis tests.
fn basisTestCandles(buf: []Candle, first: Date, count: usize) []Candle {
for (0..count) |i| {
const d = first.addDays(@intCast(i));
buf[i] = .{ .date = d, .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 };
}
return buf[0..count];
}
test "cacheCandles stamps adj_basis at the newest bar" {
// A full replace means every row carries the provider's current
// adjustment basis, which by construction covers every corporate
// action up to the newest bar.
const io = std.testing.io;
const allocator = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
var s = Store.init(io, allocator, dir_path);
var buf: [5]Candle = undefined;
const candles = basisTestCandles(&buf, Date.fromYmd(2026, 8, 10), 5);
s.cacheCandles("SMPL", candles, .{ .provider = .tiingo }, 9_999_999_999);
const meta = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
try std.testing.expect(meta.adj_basis.eql(Date.fromYmd(2026, 8, 14)));
try std.testing.expect(meta.last_date.eql(Date.fromYmd(2026, 8, 14)));
}
test "appendCandles does not advance adj_basis" {
// The core invariant. Appended bars arrive with adj_close == close
// (nothing has gone ex after them yet) and do NOT restate the rows
// already on disk, so claiming a newer basis would mask exactly the
// staleness this field exists to detect.
const io = std.testing.io;
const allocator = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
var s = Store.init(io, allocator, dir_path);
var buf: [3]Candle = undefined;
s.cacheCandles("SMPL", basisTestCandles(&buf, Date.fromYmd(2026, 6, 1), 3), .{ .provider = .tiingo }, 9_999_999_999);
const before = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
try std.testing.expect(before.adj_basis.eql(Date.fromYmd(2026, 6, 3)));
var more: [3]Candle = undefined;
s.appendCandles("SMPL", basisTestCandles(&more, Date.fromYmd(2026, 8, 12), 3), before, 9_999_999_999);
const after = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
// last_date advanced...
try std.testing.expect(after.last_date.eql(Date.fromYmd(2026, 8, 14)));
// ...but the basis is pinned to the last full fetch.
try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 3)));
}
test "legacy candles_meta without adj_basis parses to the epoch sentinel" {
// Same migration story as tiingo_retry_after_s, and the stakes are
// higher: making this field required would route every cached symbol
// through the cold-start path, which on a 404 writes a negative
// marker OVER candles_daily.srf and destroys the history.
const allocator = std.testing.allocator;
const legacy =
\\#!srfv1
\\#!expires=1787000100
\\#!created=1786748010
\\last_close:num:776.34,last_date::2026-08-14,provider::yahoo
\\
;
const parsed = try Store.deserializeCandleMeta(allocator, legacy);
try std.testing.expect(parsed.adj_basis.eql(Date.epoch));
// A sentinel basis on a dividend payer is exactly the "needs
// restatement" signal, so it must not be mistaken for "current".
try std.testing.expect(parsed.adj_basis.lessThan(Date.fromYmd(2026, 6, 18)));
}
test "adj_basis is elided when unset and round-trips when set" {
const allocator = std.testing.allocator;
const unset = Store.CandleMeta{
.last_close = 100.0,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .tiingo,
};
const bare = try Store.serializeCandleMeta(std.testing.io, allocator, unset, .{ .expires = 1 });
defer allocator.free(bare);
try std.testing.expect(std.mem.indexOf(u8, bare, "adj_basis") == null);
var set = unset;
set.adj_basis = Date.fromYmd(2026, 8, 14);
const full = try Store.serializeCandleMeta(std.testing.io, allocator, set, .{ .expires = 1 });
defer allocator.free(full);
try std.testing.expect(std.mem.indexOf(u8, full, "adj_basis::2026-08-14") != null);
const parsed = try Store.deserializeCandleMeta(allocator, full);
try std.testing.expect(parsed.adj_basis.eql(Date.fromYmd(2026, 8, 14)));
}
// ── writeRaw / appendRaw atomicity ───────────────────────────
//
// A concurrent reader hitting a cache file mid-write must never see a

21
src/commands/diagnose.zig
View file

@ -248,6 +248,27 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
try out.print(", Tiingo backoff lapsed (retries next refresh)", .{});
}
try out.print("\n", .{});
// Adjustment basis. The cache is append-only, so a distribution
// that goes ex after the last full fetch never marks down the
// bars behind it - total returns then read low by roughly the
// missed yield. Report it here because the symptom (a slightly
// low 1Y total return) is otherwise invisible.
if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| {
if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) {
try out.print(
"adj basis {f} - STALE, {f} went ex behind it; total returns read low until restated\n",
.{ m.meta.adj_basis, newest_action },
);
} else {
try out.print(
"adj basis {f} - current through the newest corporate action ({f})\n",
.{ m.meta.adj_basis, newest_action },
);
}
} else {
try out.print("adj basis {f} - no dividends or splits cached, nothing to restate\n", .{m.meta.adj_basis});
}
} else {
try out.print("local nothing cached\n", .{});
}

259
src/service.zig
View file

@ -23,6 +23,7 @@ const Holding = @import("models/etf_profile.zig").Holding;
const SectorWeight = @import("models/etf_profile.zig").SectorWeight;
const Config = @import("Config.zig");
const cache = @import("cache/store.zig");
const freshness = @import("cache/freshness.zig");
const srf = @import("srf");
const analysis = @import("analytics/analysis.zig");
const transaction_log = @import("models/transaction_log.zig");
@ -677,6 +678,112 @@ pub const DataService = struct {
return triple;
}
/// Fixed start date for any full-history candle fetch. See
/// `populateAllFromTiingo`'s doc comment for the rationale.
const full_history_start: Date = Date.fromYmd(2000, 1, 1);
/// Replace a symbol's entire candle series, restating `adj_close`
/// from whichever provider will serve it.
///
/// This is the only way a cached series' adjustment basis ever
/// advances - see `CandleMeta.adj_basis`. Incremental appends
/// cannot restate the rows already on disk, so once a distribution
/// goes ex behind them the whole series reads low until something
/// rewrites it. That something is this function.
///
/// Tiingo first (it returns dividends and splits in the same
/// response, and its `adj_close` is the one the analytics layer is
/// written against), falling back to a full-range Yahoo fetch when
/// Tiingo does not carry the symbol.
///
/// **Never writes a negative-cache entry.** Callers reach this
/// holding a working series; `writeNegative` would overwrite
/// `candles_daily.srf` with a marker and destroy that history. A
/// restatement that cannot be completed is a "try again later", not
/// a verdict about the symbol. Callers should treat any error as
/// "keep the existing series and carry on".
///
/// Returns `error.NotFound` only when *every* provider says it does
/// not carry the symbol - that is the one outcome a cold-start
/// caller may legitimately turn into a negative-cache entry. Any
/// other failure surfaces as `FetchFailed` / `TransientError` /
/// `AuthError` so a network blip can never be mistaken for
/// "this symbol does not exist".
fn refetchFullHistory(
self: *DataService,
symbol: []const u8,
today: Date,
now_s: i64,
prefer_yahoo: bool,
) (DataError || error{NotFound})![]Candle {
self.assertNetworkAllowed("getCandles refetchFullHistory");
const kind = market.classify(symbol);
var s = self.store();
// Tracks whether each provider affirmatively said "no such
// symbol", as opposed to failing for some other reason. Only
// unanimous 404s justify the caller writing a negative entry.
var tiingo_not_found = false;
if (!prefer_yahoo) {
if (self.populateAllFromTiingo(symbol)) |triple| {
defer Dividend.freeSlice(self.allocator, triple.dividends);
defer self.allocator.free(triple.splits);
if (triple.candles.len > 0) return triple.candles;
// An empty full-history response is not a restatement;
// fall through rather than caching emptiness.
self.allocator.free(triple.candles);
log.warn("{s}: Tiingo full history returned no bars, trying Yahoo", .{symbol});
} else |err| {
// Transient failures must not silently degrade to a
// second provider - the caller needs to know to retry.
if (err == error.RateLimited or isTransientError(err)) return DataError.TransientError;
if (err == error.Unauthorized) {
log.err("{s}: Tiingo auth failed during restatement - check TIINGO_API_KEY", .{symbol});
return DataError.AuthError;
}
tiingo_not_found = isPermanentProviderFailure(err);
log.info("{s}: Tiingo cannot serve full history ({s}), trying Yahoo", .{ symbol, @errorName(err) });
}
}
// Yahoo fallback. `fetchCandles` takes an arbitrary range, so a
// full-history request is just a wide one. Yahoo carries no
// dividend/split endpoint here; those caches are Polygon-primary
// and merged separately, so leaving them untouched is correct.
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, full_history_start, today)) |candles| {
if (candles.len > 0) {
s.cacheCandles(symbol, candles, .{
.provider = .yahoo,
// Preserve the Tiingo verdict this call just
// learned, so the next pass doesn't re-probe.
.tiingo_retry_after_s = if (tiingo_not_found)
cache.computeExpires(
now_s,
.{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct },
symbol,
)
else
0,
}, expiryAfterFetch(now_s, kind, candles));
log.info("{s}: full history restated from Yahoo ({d} bars)", .{ symbol, candles.len });
return candles;
}
self.allocator.free(candles);
log.warn("{s}: Yahoo full history returned no bars", .{symbol});
} else |err| {
log.warn("{s}: Yahoo full history failed: {s}", .{ symbol, @errorName(err) });
// Both providers affirmatively disclaim the symbol.
if (tiingo_not_found and isPermanentProviderFailure(err)) return error.NotFound;
}
} else |_| {
log.warn("{s}: Yahoo provider not available for full history", .{symbol});
}
return DataError.FetchFailed;
}
/// Invalidate cached data for a symbol so the next get* call forces a fresh fetch.
pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void {
var s = self.store();
@ -1032,7 +1139,17 @@ pub const DataService = struct {
// (Force-refresh skips server sync too: the user explicitly
// asked for fresh provider data.)
if (!opts.force_refresh and self.syncCandlesFromServer(symbol)) {
if (s.isCandleMetaFresh(symbol)) {
// Re-read meta: the sync wrote the server's bytes
// verbatim, so its view of both freshness AND
// adjustment basis is now ours. `serverBarRegression`
// only guards `last_date` going backwards - it cannot
// see a same-dated file whose historical adj_close is
// stale, so the basis has to be re-checked here.
const synced = if (s.readCandleMeta(symbol)) |sm| sm.meta else m;
const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol);
if (s.isCandleMetaFresh(symbol) and
!freshness.adjustmentBasisStale(synced.adj_basis, synced.last_date, synced_action))
{
log.debug("{s}: candles synced from server and fresh", .{symbol});
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
@ -1047,6 +1164,36 @@ pub const DataService = struct {
// this stale path (next post-close / NAV-availability time).
const expires = market.nextCandleExpiry(now_s, kind);
// A corporate action has gone ex since the basis that
// produced this series, so the cached `adj_close` values
// behind it were never marked down. Appending cannot fix
// that - only replacing the file can. Do this BEFORE the
// incremental fetch so a symbol that needs both a top-up
// and a restatement costs one full fetch, not an append
// followed by a second pass.
//
// Deliberately not checked on the fresh-cache path above:
// the candle TTL lapses at least once per trading day, so
// detection is at most a day behind, and paying a
// dividend/split cache read on every getCandles call would
// tax the hot portfolio-pricing path for nothing.
if (freshness.adjustmentBasisStale(
m.adj_basis,
m.last_date,
freshness.newestCorporateAction(self.allocator, &s, symbol),
)) {
log.info("{s}: restating full history (adj_basis {f} predates a corporate action)", .{ symbol, m.adj_basis });
if (self.refetchFullHistory(symbol, today, now_s, now_s < m.tiingo_retry_after_s)) |candles| {
return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
} else |err| {
// Restatement is best-effort. The existing series
// is untouched and still usable, just understated
// by the missed adjustment - fall through to the
// normal top-up and retry on the next stale pass.
log.warn("{s}: full-history restatement failed ({s}); adjustment basis remains stale, falling back to incremental append", .{ symbol, @errorName(err) });
}
}
// Only skip the fetch when we already hold the latest
// *available* bar (weekend/holiday/pre-close gap, or
// caught up): just bump the TTL. Gating on the
@ -1140,19 +1287,27 @@ pub const DataService = struct {
log.debug("{s}: candles synced from server but stale, falling through to full fetch", .{symbol});
}
// No usable cache - full fetch via the orchestrated Tiingo
// helper, which writes candles + dividends + splits caches in
// one shot from a single HTTP response. The fixed start date
// (see `populateAllFromTiingo`) is 2000-01-01, deep enough to
// cover a 10Y trailing-return window even when `--as-of`
// back-dates the reference into 2014-era imported portfolio
// history, plus a buffer for older corporate actions like
// SPYM's 2017-10-16 split.
// No usable cache - full fetch. Tiingo first (it returns
// candles + dividends + splits from one response), falling back
// to a full-range Yahoo fetch when Tiingo does not carry the
// symbol. The fixed start date (see `populateAllFromTiingo`) is
// 2000-01-01, deep enough to cover a 10Y trailing-return window
// even when `--as-of` back-dates the reference into 2014-era
// imported portfolio history, plus a buffer for older corporate
// actions like SPYM's 2017-10-16 split.
//
// The Yahoo fallback matters here beyond convenience: this
// branch used to be Tiingo-only, so a symbol Tiingo does not
// carry could not be cold-started at all - it 404'd, wrote a
// negative-cache marker *over* candles_daily.srf, and stayed
// unavailable until `cache clear`. Any symbol that reached this
// branch with a populated cache would have had its history
// destroyed.
log.debug("{s}: fetching full candle history from provider", .{symbol});
self.assertNetworkAllowed("getCandles full populateAllFromTiingo");
const triple = self.populateAllFromTiingo(symbol) catch |err| {
if (err == error.RateLimited or err == error.ServerError or err == error.RequestFailed) {
const prior_backoff: i64 = if (meta_result) |mr| mr.meta.tiingo_retry_after_s else 0;
const candles = self.refetchFullHistory(symbol, today, now_s, now_s < prior_backoff) catch |err| {
if (err == DataError.TransientError) {
// Transient: increment fail_count on existing meta so
// we know to back off if this keeps happening.
if (meta_result) |mr| {
@ -1162,24 +1317,20 @@ pub const DataService = struct {
}
return DataError.TransientError;
}
// Only a genuine NotFound means "this symbol has no candle
// data on Tiingo" (the sole historical-candle provider since
// the 2026-05 audit) and earns a sticky negative-cache entry.
// Unauthorized / InvalidResponse / PaymentRequired and the
// like are not permanent facts about the symbol (transient
// auth misconfig, a malformed response) - fail this call but
// stay retryable, matching fetchCached's policy. This matters
// now that the candle negative cache is actually honored: a
// bad negative would otherwise stick until --refresh.
if (isPermanentProviderFailure(err)) s.writeNegative(symbol, .candles_daily);
// Only a unanimous NotFound - every provider affirmatively
// disclaims the symbol - earns a sticky negative-cache
// entry. Auth trouble, a malformed response, or a Yahoo
// network blip are not permanent facts about the symbol, so
// they fail this call but stay retryable. This matters
// because the negative marker replaces the candle file: a
// bad negative both suppresses retries until `--refresh`
// and throws away whatever history was cached.
if (err == error.NotFound) s.writeNegative(symbol, .candles_daily);
if (err == DataError.AuthError) return DataError.AuthError;
return DataError.FetchFailed;
};
// populateAllFromTiingo writes all three caches itself; we
// free the slices we don't return.
defer Dividend.freeSlice(self.allocator, triple.dividends);
defer self.allocator.free(triple.splits);
return .{ .data = triple.candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
}
/// Fetch dividend history for a symbol.
@ -4079,8 +4230,58 @@ test "loadAllPrices offline mode skips network and returns cached" {
try std.testing.expectEqual(@as(usize, 1), result.failed_count);
}
// ── Tiingo coverage bookkeeping ──────────────────────────────
//
// ── adjustment-basis restatement ─────────────────────────────
test "getCandles offline never escalates a stale adjustment basis" {
// Restatement does network I/O, so `skip_network` must return the
// cached series untouched rather than escalating. Pinned with
// `panic_on_network_attempt`, which fires inside
// `refetchFullHistory`'s `assertNetworkAllowed` if this regresses.
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
const config = Config{ .cache_dir = dir_path };
var svc = DataService.init(io, allocator, config);
defer svc.deinit();
var store = svc.store();
var candles = [_]Candle{
.{ .date = Date.fromYmd(2026, 8, 12), .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 },
.{ .date = Date.fromYmd(2026, 8, 13), .open = 11, .high = 11, .low = 11, .close = 11, .adj_close = 11, .volume = 1 },
};
store.cacheCandles("SMPL", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Roll the basis back behind a cached ex-date so the series is
// unambiguously due for restatement.
var divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 7, 1), .amount = 1.0 }};
store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
var meta = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta;
meta.adj_basis = Date.fromYmd(2026, 6, 1);
store.updateCandleMeta("SMPL", meta, 1); // expiry in the past => stale
try std.testing.expect(freshness.adjustmentBasisStale(
meta.adj_basis,
meta.last_date,
freshness.newestCorporateAction(allocator, &store, "SMPL"),
));
svc.panic_on_network_attempt = true;
const result = try svc.getCandles("SMPL", .{ .skip_network = true });
defer result.deinit();
// Served from cache, series intact, basis untouched.
try std.testing.expectEqual(@as(usize, 2), result.data.len);
const after = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta;
try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 1)));
// And critically, no negative-cache marker was written over the
// candle file.
try std.testing.expect(!store.isNegative("SMPL", .candles_daily));
}
// ── Tiingo coverage bookkeeping ──────────────────────────────//
// Regression suite for the one-way drift to Yahoo. The old code
// routed on `CandleMeta.provider == .yahoo`, so any non-transient
// Tiingo failure latched permanently: Yahoo was tried first,