Compare commits

...

6 commits

Author SHA1 Message Date
e3ce8f32a1
build portfolio risk from a total-return index
All checks were successful
Generic zig build / build (push) Successful in 5m10s
Generic zig build / publish-macos (push) Successful in 11s
Generic zig build / deploy (push) Successful in 17s
2026-08-17 20:01:06 -07:00
337d346421
fix stale provider doc comments 2026-08-17 19:43:11 -07:00
6674969001
consolidate returns calculations 2026-08-17 19:39:45 -07:00
7163610b8e
restate adj_close when a corporate action goes ex 2026-08-17 18:26:36 -07:00
c4680dbe45
redact sensitive url data in logs 2026-08-17 16:55:24 -07:00
6c75ebf1ec
stop convergence to Yahoo for equity coverage 2026-08-17 16:48:39 -07:00
15 changed files with 2033 additions and 354 deletions

View file

@ -56,7 +56,7 @@ pub const BenchmarkComparison = struct {
/// Per-position data needed for weighted return calculations.
/// `returns.week` carries the 1-week return, populated automatically
/// by `performance.trailingReturns`.
/// by `performance.totalReturns`.
pub const PositionReturn = struct {
symbol: []const u8,
weight: f64,
@ -338,7 +338,7 @@ pub fn buildComparison(
) BenchmarkComparison {
// `stock_trailing.week` and `bond_trailing.week` propagate
// through `toReturnsByPeriod` automatically - see
// `performance.trailingReturns`, which populates the field
// `performance.totalReturns`, which populates the field
// alongside the longer trailing periods.
const stock_r = toReturnsByPeriod(stock_trailing);
const bond_r = toReturnsByPeriod(bond_trailing);
@ -621,7 +621,7 @@ test "conservativeWeightedReturn single position single period" {
test "buildComparison with week returns" {
// Week returns now flow through `TrailingReturns.week` rather
// than separate parameters - `performance.trailingReturns`
// than separate parameters - `performance.totalReturns`
// populates the field automatically.
const stock_tr = TrailingReturns{
.one_year = makePR(0.20, 0.20),

View file

@ -1,6 +1,7 @@
const std = @import("std");
const Date = @import("../Date.zig");
const Candle = @import("../models/candle.zig").Candle;
const sliceCandlesAsOf = @import("../models/candle.zig").sliceCandlesAsOf;
const Dividend = @import("../models/dividend.zig").Dividend;
const Split = @import("../models/split.zig").Split;
const cumulativeSplitRatio = @import("../models/split.zig").cumulativeSplitRatio;
@ -161,7 +162,12 @@ pub const TrailingReturns = struct {
/// - When dividend data is complete: reinvestment >= adj_close (compounding)
/// - When dividend data is incomplete: adj_close > reinvestment (missing dividends)
/// So the higher value is always the more correct one.
pub fn withDividendFallback(div_returns: TrailingReturns, adj_close_returns: TrailingReturns) TrailingReturns {
///
/// Private: reachable only through `totalReturns` /
/// `totalReturnsMonthEnd`, which is the point - the merge is not
/// optional, and a caller able to skip it can produce a total return
/// that is quietly low whenever the cached adjustment basis is stale.
fn withDividendFallback(div_returns: TrailingReturns, adj_close_returns: TrailingReturns) TrailingReturns {
return .{
.one_year = bestResult(div_returns.one_year, adj_close_returns.one_year),
.three_year = bestResult(div_returns.three_year, adj_close_returns.three_year),
@ -183,9 +189,77 @@ fn bestResult(a: ?PerformanceResult, b: ?PerformanceResult) ?PerformanceResult {
return if (a_ann >= b_ann) a else b;
}
// Total return (the public entry points)
//
// There are two ways to compute a total return from cached data and
// neither is reliable alone:
//
// - The `adj_close` ratio inherits whatever adjustment basis the
// cache holds. When a distribution has gone ex since the last full
// fetch, the bars behind it were never marked down, so the ratio
// reads low by roughly the missed yield (see `CandleMeta.adj_basis`).
// - Simulated reinvestment over raw `close` is exact, but only if the
// dividend record is complete. A missing dividend silently drops
// from the compounding.
//
// The failure modes are opposite and both are downward, so taking the
// higher of the two is correct in both directions. That merge is the
// only sanctioned way to get a total return, which is why these are the
// only public entry points and the four single-engine wrappers below
// are private: a caller reaching for the adj_close series alone gets a
// number that is quietly wrong exactly when the cache is stale.
//
// Both take `as_of` and truncate internally. Previously the as-of
// behaviour depended on whether the caller remembered to pre-slice its
// candles, which two of three call sites did and one did not.
/// Total return over 1/3/5/10-year windows ending at the newest bar on
/// or before `as_of`.
///
/// Pass `dividends = null` when no dividend record is available; the
/// result then falls back to the adj_close series alone, which is the
/// best available answer but is exposed to adjustment-basis staleness.
pub fn totalReturns(
candles: []const Candle,
dividends: ?[]const Dividend,
as_of: Date,
) TrailingReturns {
const window = sliceCandlesAsOf(candles, as_of);
if (window.len == 0) return .{};
const adj = trailingReturns(window);
const divs = dividends orelse return adj;
return withDividendFallback(trailingReturnsWithDividends(window, divs), adj);
}
/// Morningstar-style total return using month-end reference dates: the
/// window ends on the last trading day of the month before `as_of`.
///
/// Same dividend-vs-adj_close merge as `totalReturns`. Note the start
/// dates snap *backward* here (Morningstar's convention) where
/// `totalReturns` snaps forward, and `.week` is left null because a
/// month-end reference makes a 7-day window meaningless.
pub fn totalReturnsMonthEnd(
candles: []const Candle,
dividends: ?[]const Dividend,
as_of: Date,
) TrailingReturns {
const window = sliceCandlesAsOf(candles, as_of);
if (window.len == 0) return .{};
const adj = trailingReturnsMonthEnd(window, as_of);
const divs = dividends orelse return adj;
return withDividendFallback(trailingReturnsMonthEndWithDividends(window, divs, as_of), adj);
}
/// Trailing returns from exact calendar date N years ago to latest candle date.
/// Start dates snap forward to the next trading day (e.g., weekend -> Monday).
pub fn trailingReturns(candles: []const Candle) TrailingReturns {
///
/// Private: the adj_close-only series understates total return whenever the
/// cached adjustment basis predates a distribution, so nothing outside this
/// module should consume it directly. Use `totalReturns`, which pairs it with
/// the dividend-reinvestment series. See that function's doc comment.
fn trailingReturns(candles: []const Candle) TrailingReturns {
if (candles.len == 0) return .{};
const end_date = candles[candles.len - 1].date;
@ -200,7 +274,10 @@ pub fn trailingReturns(candles: []const Candle) TrailingReturns {
}
/// Same as trailingReturns but with dividend reinvestment.
pub fn trailingReturnsWithDividends(
///
/// Private: use `totalReturns`, which merges this with the adj_close
/// series so an incomplete dividend record cannot understate the result.
fn trailingReturnsWithDividends(
candles: []const Candle,
dividends: []const Dividend,
) TrailingReturns {
@ -221,7 +298,9 @@ pub fn trailingReturnsWithDividends(
/// End date = last calendar day of prior month. Start date = that month-end minus N years.
/// Both dates snap backward to the last trading day on or before, matching
/// Morningstar's "last business day of the month" convention.
pub fn trailingReturnsMonthEnd(candles: []const Candle, as_of: Date) TrailingReturns {
///
/// Private: use `totalReturnsMonthEnd`. See `totalReturns`.
fn trailingReturnsMonthEnd(candles: []const Candle, as_of: Date) TrailingReturns {
if (candles.len == 0) return .{};
// End reference = last day of the prior month (snaps backward to last trading day)
@ -236,7 +315,9 @@ pub fn trailingReturnsMonthEnd(candles: []const Candle, as_of: Date) TrailingRet
}
/// Same as trailingReturnsMonthEnd but with dividend reinvestment.
pub fn trailingReturnsMonthEndWithDividends(
///
/// Private: use `totalReturnsMonthEnd`. See `totalReturns`.
fn trailingReturnsMonthEndWithDividends(
candles: []const Candle,
dividends: []const Dividend,
as_of: Date,
@ -1002,3 +1083,143 @@ test "trailingReturnsPriceOnly -- price-only diverges from total return on divid
// above demonstrates.)
try std.testing.expectApproxEqAbs(@as(f64, 0.0), total.one_year.?.total_return, 0.01);
}
// totalReturns / totalReturnsMonthEnd (the assemblers)
//
// These replaced three divergent call-site assemblies: `service.zig`
// merged adj_close with dividends for both as-of and month-end,
// `views/review.zig` did the same for month-end only, and
// `views/projections.zig` did no merge at all - it called the
// adj_close-only series directly, so a stale adjustment basis flowed
// straight into the benchmark comparison and the Monte Carlo inputs.
/// 500 consecutive daily bars at a flat price, so a price return of
/// exactly zero isolates the dividend contribution.
fn flatSeries(buf: []Candle, first: Date, price: f64) []Candle {
for (buf, 0..) |*c, i| c.* = makeCandle(first.addDays(@intCast(i)), price);
return buf;
}
test "totalReturns: empty candles yields an empty struct" {
const tr = totalReturns(&.{}, null, Date.fromYmd(2026, 6, 1));
try std.testing.expect(tr.one_year == null);
try std.testing.expect(tr.three_year == null);
try std.testing.expect(tr.five_year == null);
try std.testing.expect(tr.ten_year == null);
try std.testing.expect(tr.week == null);
}
test "totalReturns: null dividends degrades to the adj_close series" {
// The no-dividend-record case must still produce a number rather
// than null - it is the best available answer, just exposed to
// adjustment-basis staleness.
var buf: [500]Candle = undefined;
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
const tr = totalReturns(candles, null, Date.fromYmd(2026, 8, 14));
try std.testing.expect(tr.one_year != null);
// Flat price, no dividends: zero total return.
try std.testing.expectApproxEqAbs(@as(f64, 0), tr.one_year.?.total_return, 1e-9);
}
test "totalReturns: recovers a distribution the adjustment basis missed" {
// The regression this whole change exists for. A series whose
// adj_close was never restated has adj_close == close throughout, so
// the adj_close ratio equals the *price* return and silently omits
// the distribution. The dividend-reinvestment series still sees it,
// and the merge must prefer that.
var buf: [500]Candle = undefined;
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
const as_of = Date.fromYmd(2026, 8, 14);
// $1.00 on a $100 close = a 1% yield, inside the 1Y window.
const divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 1, 15), .amount = 1.0 }};
const stale = totalReturns(candles, null, as_of);
const merged = totalReturns(candles, &divs, as_of);
// Uncorrected, the distribution is invisible.
try std.testing.expectApproxEqAbs(@as(f64, 0), stale.one_year.?.total_return, 1e-9);
// Merged, it is worth exactly its yield.
try std.testing.expectApproxEqAbs(@as(f64, 0.01), merged.one_year.?.total_return, 1e-9);
try std.testing.expect(merged.one_year.?.total_return > stale.one_year.?.total_return);
}
test "totalReturns: a complete dividend record is never worse than adj_close" {
// `withDividendFallback` takes the higher of the two per period, so
// adding dividends can only raise a total return, never lower it.
// That directionality is what makes the merge safe: both failure
// modes (stale basis, incomplete dividends) bias downward.
var buf: [500]Candle = undefined;
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
const as_of = Date.fromYmd(2026, 8, 14);
const divs = [_]Dividend{
.{ .ex_date = Date.fromYmd(2025, 9, 15), .amount = 0.5 },
.{ .ex_date = Date.fromYmd(2026, 3, 15), .amount = 0.5 },
};
const without = totalReturns(candles, null, as_of);
const with = totalReturns(candles, &divs, as_of);
try std.testing.expect(with.one_year.?.total_return >= without.one_year.?.total_return);
}
test "totalReturns: as_of truncates the window instead of using the newest bar" {
// Previously the as-of behaviour depended on whether the caller
// remembered to pre-slice - `views/projections.zig` did,
// `service.zig` did not. Now it is the function's job.
var buf: [500]Candle = undefined;
// Rising 1/day from 100, so the endpoint is identifiable by value.
for (&buf, 0..) |*c, i| {
c.* = makeCandle(Date.fromYmd(2025, 6, 1).addDays(@intCast(i)), 100.0 + @as(f64, @floatFromInt(i)));
}
const candles = buf[0..];
const late = totalReturns(candles, null, Date.fromYmd(2026, 8, 14));
const early = totalReturns(candles, null, Date.fromYmd(2026, 6, 14));
// Both resolve a 1Y window...
try std.testing.expect(late.one_year != null);
try std.testing.expect(early.one_year != null);
// ...ending on their own as_of, not on the last bar in the slice.
try std.testing.expect(late.one_year.?.to.eql(Date.fromYmd(2026, 8, 14)));
try std.testing.expect(early.one_year.?.to.eql(Date.fromYmd(2026, 6, 14)));
}
test "totalReturns: as_of before all candles yields an empty struct" {
var buf: [500]Candle = undefined;
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
const tr = totalReturns(candles, null, Date.fromYmd(2020, 1, 1));
try std.testing.expect(tr.one_year == null);
try std.testing.expect(tr.week == null);
}
test "totalReturnsMonthEnd: window ends on the last bar of the prior month" {
// Morningstar convention, and the reason this variant exists
// separately: the endpoint is a month boundary, not `as_of` itself.
var buf: [500]Candle = undefined;
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
const tr = totalReturnsMonthEnd(candles, null, Date.fromYmd(2026, 8, 14));
try std.testing.expect(tr.one_year != null);
try std.testing.expect(tr.one_year.?.to.eql(Date.fromYmd(2026, 7, 31)));
// A month-end reference makes a 7-day window meaningless.
try std.testing.expect(tr.week == null);
}
test "totalReturnsMonthEnd: recovers a missed distribution too" {
var buf: [500]Candle = undefined;
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
const as_of = Date.fromYmd(2026, 8, 14);
const divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 1, 15), .amount = 1.0 }};
const stale = totalReturnsMonthEnd(candles, null, as_of);
const merged = totalReturnsMonthEnd(candles, &divs, as_of);
try std.testing.expectApproxEqAbs(@as(f64, 0), stale.one_year.?.total_return, 1e-9);
try std.testing.expectApproxEqAbs(@as(f64, 0.01), merged.one_year.?.total_return, 1e-9);
}
test "totalReturnsMonthEnd: empty candles yields an empty struct" {
const tr = totalReturnsMonthEnd(&.{}, null, Date.fromYmd(2026, 6, 1));
try std.testing.expect(tr.one_year == null);
try std.testing.expect(tr.ten_year == null);
}

View file

@ -36,6 +36,8 @@ const std = @import("std");
const Date = @import("../Date.zig");
const Candle = @import("../models/candle.zig").Candle;
const risk = @import("risk.zig");
const Dividend = @import("../models/dividend.zig").Dividend;
const Split = @import("../models/split.zig").Split;
/// Per-window flags marking metrics that required holding-dropout
/// renormalization. Set when at least one position lacked candle coverage
@ -87,8 +89,36 @@ pub const PositionCandles = struct {
/// Position weight in the portfolio (market_value / total_value).
/// Must be non-negative; zero-weight positions are skipped entirely.
weight: f64,
/// Cash distributions, **newest first** - the order
/// `cache.Store.read` returns them in, since `writeMerged` sorts
/// descending by date.
///
/// When this and `splits` are both empty, the resample falls back to
/// the provider's `adj_close`, preserving the behaviour callers had
/// before these fields existed. Supply them to get a total-return
/// index built from raw `close` instead - see `synthesizeWindow`.
dividends: []const Dividend = &.{},
/// Split events, **newest first**, same convention as `dividends`.
///
/// Required for correctness whenever `dividends` is supplied: raw
/// `close` is not split-adjusted, so an index built without these
/// would read a 2:1 split as a -50% month.
splits: []const Split = &.{},
};
/// Debug-only guard on the newest-first ordering the resample relies on.
/// A mis-ordered slice would silently mis-time reinvestment rather than
/// fail, so catch it where tests will see it.
fn assertDescending(comptime T: type, items: []const T, comptime dateField: []const u8) void {
if (!std.debug.runtime_safety) return;
if (items.len < 2) return;
for (items[1..], 0..) |item, i| {
const newer = @field(items[i], dateField);
const older = @field(item, dateField);
std.debug.assert(!newer.lessThan(older));
}
}
/// Compute true portfolio-level risk metrics for the standard windows.
/// Iterates `positions`, derives per-position monthly return series,
/// builds a weighted synthetic series per window with dropout-and-
@ -314,20 +344,73 @@ fn synthesizeWindow(
for (participants_buf[0..n_participants], 0..) |orig_idx, p_idx| {
const cand = positions[orig_idx].candles;
// Walk candles, recording the LAST close in each month bucket
const divs = positions[orig_idx].dividends;
const spls = positions[orig_idx].splits;
assertDescending(Dividend, divs, "ex_date");
assertDescending(Split, spls, "date");
// Which price series to resample.
//
// With corporate actions supplied, build a total-return index in
// units of shares held: start at one share, multiply on a split,
// and buy more with each distribution at that day's close.
//
// value(t) = shares(t) * close(t)
//
// Both events pass through that expression continuously - a split
// multiplies shares while dividing close, and a distribution buys
// exactly the shares its per-share drop paid for - so the series
// has no artificial cliffs and month-over-month ratios are true
// total returns.
//
// Why not `adj_close`: it is only as current as the last full
// candle fetch (see `CandleMeta.adj_basis`), and Yahoo's parser
// leaves it at 0 for a JSON `null`, which the `prev_p <= 0` guard
// below then silently drops - losing a whole month from the
// series rather than merely mis-stating it.
//
// Without corporate actions there is nothing to build from, so
// fall back to `adj_close` (via `chartClose`, which handles the
// zero case). That keeps callers predating these fields on
// exactly their old behaviour.
const use_index = divs.len > 0 or spls.len > 0;
// Walk candles, recording the LAST value in each month bucket
// that falls within the window. We use yearMonth() for the
// month-boundary comparison (cheap integer compare) and
// Date.monthsBetween for the slot index into `prices`.
var prev_ym: u32 = 0;
var prev_date: Date = window_start;
var last_close: f64 = 0;
var last_value: f64 = 0;
var have_any = false;
// Corporate-action cursors. Both slices are newest-first, so walk
// them from the tail to consume events chronologically. Pre-window
// bars are walked too, which is what makes `shares` correct on
// entry to the window.
var shares: f64 = 1.0;
var di: usize = divs.len;
var si: usize = spls.len;
for (cand) |c| {
if (use_index) {
// Splits first: a distribution's per-share amount is
// quoted against the share count in force on its ex-date.
while (si > 0 and !c.date.lessThan(spls[si - 1].date)) {
shares *= spls[si - 1].ratio();
si -= 1;
}
while (di > 0 and !c.date.lessThan(divs[di - 1].ex_date)) {
if (c.close > 0) shares += divs[di - 1].amount * shares / c.close;
di -= 1;
}
}
const value = if (use_index) shares * c.close else c.chartClose();
if (c.date.days < window_start.days) {
// Before window: still track the running last-close,
// Before window: still track the running last value,
// but only emit it once we cross into the window.
last_close = c.adj_close;
last_value = value;
prev_ym = c.date.yearMonth();
prev_date = c.date;
have_any = true;
@ -336,27 +419,27 @@ fn synthesizeWindow(
if (c.date.days > window_end.days) break;
const ym = c.date.yearMonth();
if (have_any and ym != prev_ym) {
// Month boundary: stash prev month's last close.
// Month boundary: stash prev month's last value.
const m_idx_signed = Date.monthsBetween(window_start, prev_date);
if (m_idx_signed >= 0) {
const m_idx: usize = @intCast(m_idx_signed);
if (m_idx < n_months_total) {
prices[p_idx * n_months_total + m_idx] = last_close;
prices[p_idx * n_months_total + m_idx] = last_value;
}
}
}
last_close = c.adj_close;
last_value = value;
prev_ym = ym;
prev_date = c.date;
have_any = true;
}
// Final partial month: stash whatever the latest close was.
// Final partial month: stash whatever the latest value was.
if (have_any) {
const m_idx_signed = Date.monthsBetween(window_start, prev_date);
if (m_idx_signed >= 0) {
const m_idx: usize = @intCast(m_idx_signed);
if (m_idx < n_months_total) {
prices[p_idx * n_months_total + m_idx] = last_close;
prices[p_idx * n_months_total + m_idx] = last_value;
}
}
}
@ -702,3 +785,173 @@ test "syntheticPortfolioRisk: return_3y annualizes correctly" {
try testing.expect(r.return_3y.? > 0.10);
try testing.expect(r.return_3y.? < 0.16);
}
// Total-return index (dividends + splits)
//
// The resample used to read `adj_close` directly. That inherits any
// staleness in the cached adjustment basis, and Yahoo's parser leaves
// `adj_close` at 0 for a JSON `null`, which the `prev_p <= 0` guard
// drops - losing the whole month. Supplying dividends and splits
// switches to an index built from raw `close`, in units of shares held.
/// Build a candle slice with an explicit close per month, so tests can
/// place a corporate action against a known price.
fn monthlyCloses(allocator: std.mem.Allocator, start_year: i16, closes: []const f64) ![]Candle {
var out = std.ArrayList(Candle).empty;
errdefer out.deinit(allocator);
for (closes, 0..) |px, i| {
const year_offset: i16 = @intCast(i / 12);
const m_in_year: u8 = @intCast((i % 12) + 1);
try out.append(allocator, makeCandle(Date.fromYmd(start_year + year_offset, m_in_year, 15), px));
try out.append(allocator, makeCandle(Date.fromYmd(start_year + year_offset, m_in_year, 28), px));
}
return out.toOwnedSlice(allocator);
}
test "synthesizeWindow index: a split is not a -50% month" {
// The correctness trap in building an index from raw `close`.
// `close` is not split-adjusted, so a 2:1 split halves it; without
// multiplying the share count the series reads a catastrophic month
// and inflates vol. 40 flat months with a split in the middle must
// produce zero volatility.
const a = testing.allocator;
var closes: [40]f64 = undefined;
for (&closes, 0..) |*c, i| c.* = if (i < 20) 200.0 else 100.0;
const cand = try monthlyCloses(a, 2022, &closes);
defer a.free(cand);
// Split lands on the first bar of the halved run.
const splits = [_]Split{.{ .date = Date.fromYmd(2023, 9, 15), .numerator = 2, .denominator = 1 }};
const positions = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .splits = &splits },
};
const r = try syntheticPortfolioRisk(a, &positions, Date.fromYmd(2025, 5, 1));
try testing.expect(r.vol_3y != null);
// A flat total-return series has no volatility. Without the share
// multiplication this would be enormous.
try testing.expect(r.vol_3y.? < 0.01);
try testing.expect(r.return_3y != null);
try testing.expectApproxEqAbs(@as(f64, 0), r.return_3y.?, 0.01);
}
test "synthesizeWindow index: distributions raise the total return" {
// Flat price, so every bit of return comes from the distributions.
const a = testing.allocator;
var closes: [40]f64 = undefined;
for (&closes) |*c| c.* = 100.0;
const cand = try monthlyCloses(a, 2022, &closes);
defer a.free(cand);
// Newest-first, matching the cache's on-disk order.
const divs = [_]Dividend{
.{ .ex_date = Date.fromYmd(2024, 9, 15), .amount = 1.0 },
.{ .ex_date = Date.fromYmd(2024, 3, 15), .amount = 1.0 },
.{ .ex_date = Date.fromYmd(2023, 9, 15), .amount = 1.0 },
};
const bare = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0 },
};
const paying = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &divs },
};
const without = try syntheticPortfolioRisk(a, &bare, Date.fromYmd(2025, 5, 1));
const with = try syntheticPortfolioRisk(a, &paying, Date.fromYmd(2025, 5, 1));
// Flat price and no distributions: zero return.
try testing.expectApproxEqAbs(@as(f64, 0), without.return_3y.?, 1e-6);
// Three 1% reinvestments inside the window compound to ~3.03%,
// annualized over 3 years to ~1%.
try testing.expect(with.return_3y.? > 0.005);
try testing.expect(with.return_3y.? < 0.015);
}
test "synthesizeWindow index: unusable adj_close no longer drops the month" {
// Yahoo yields adj_close == 0 for a JSON null. The `prev_p <= 0`
// guard then skips both month transitions touching it, silently
// shortening the series. With an index built from `close`, the
// zeroed field is never consulted.
const a = testing.allocator;
var closes: [40]f64 = undefined;
for (&closes, 0..) |*c, i| c.* = 100.0 + @as(f64, @floatFromInt(i));
const cand = try monthlyCloses(a, 2022, &closes);
defer a.free(cand);
// Zero out adj_close on a mid-window month, as a null would.
for (cand) |*c| {
if (c.date.yearMonth() == Date.fromYmd(2024, 1, 15).yearMonth()) c.adj_close = 0;
}
const divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2023, 6, 15), .amount = 0.5 }};
const bare = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0 },
};
const indexed = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &divs },
};
const as_of = Date.fromYmd(2025, 5, 1);
const without = try syntheticPortfolioRisk(a, &bare, as_of);
const with = try syntheticPortfolioRisk(a, &indexed, as_of);
// Both still produce numbers - `chartClose`'s zero fallback keeps
// the bare path alive rather than dropping the month outright.
try testing.expect(without.return_3y != null);
try testing.expect(with.return_3y != null);
// And the indexed path is strictly better: it also counts the
// distribution the bare path cannot see.
try testing.expect(with.return_3y.? > without.return_3y.?);
}
test "synthesizeWindow index: no corporate actions preserves adj_close behaviour" {
// The defaulted fields must be a no-op, or every caller predating
// them silently changes numbers.
const a = testing.allocator;
const cand = try buildMonthlyCandles(a, 2022, 45, &linearGrowth);
defer a.free(cand);
const positions = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0 },
};
const as_of = Date.fromYmd(2025, 11, 1);
const baseline = try syntheticPortfolioRisk(a, &positions, as_of);
// Same input, explicitly-empty corporate actions.
const explicit = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &.{}, .splits = &.{} },
};
const same = try syntheticPortfolioRisk(a, &explicit, as_of);
try testing.expectEqual(baseline.return_3y, same.return_3y);
try testing.expectEqual(baseline.vol_3y, same.vol_3y);
try testing.expectEqual(baseline.sharpe_3y, same.sharpe_3y);
}
test "synthesizeWindow index: split and dividend on the same date" {
// Order matters: a distribution's per-share amount is quoted
// against the share count in force on its ex-date, so the split
// must apply first. Getting it backwards misprices the
// reinvestment by the split ratio.
const a = testing.allocator;
var closes: [40]f64 = undefined;
for (&closes, 0..) |*c, i| c.* = if (i < 20) 200.0 else 100.0;
const cand = try monthlyCloses(a, 2022, &closes);
defer a.free(cand);
const shared = Date.fromYmd(2023, 9, 15);
const splits = [_]Split{.{ .date = shared, .numerator = 2, .denominator = 1 }};
const divs = [_]Dividend{.{ .ex_date = shared, .amount = 1.0 }};
const positions = [_]PositionCandles{
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &divs, .splits = &splits },
};
const r = try syntheticPortfolioRisk(a, &positions, Date.fromYmd(2025, 5, 1));
// Flat total-return apart from a single 1% reinvestment: small
// positive return, and still essentially no volatility.
try testing.expect(r.return_3y != null);
try testing.expect(r.return_3y.? > 0);
try testing.expect(r.return_3y.? < 0.01);
try testing.expect(r.vol_3y.? < 0.02);
}

View file

@ -30,11 +30,30 @@
//!
//! Pure: dates and `now_s` in, findings out. No I/O, no cache reads, no
//! fetches. The caller supplies the corpus.
//!
//! ## Adjustment-basis staleness
//!
//! A second, unrelated staleness question lives here too:
//! `adjustmentBasisStale` asks whether a symbol's cached `adj_close`
//! values predate a corporate action they should already reflect. It is
//! the same shape of problem - "this cache entry is quietly frozen" -
//! and it follows the same two-part split as the peer analysis above:
//! `newestCorporateAction` gathers from disk (like `collect`), and
//! `adjustmentBasisStale` decides (like `scan`, pure dates in, bool
//! out).
//!
//! It deliberately does not live in `store.zig`. The cache layer
//! serializes and reads; deciding that a distribution behind the basis
//! means the series must be refetched is domain policy, and
//! `store.zig`'s own `updateCandleMeta` doc records the same boundary
//! for market-clock knowledge ("owned by the caller").
const std = @import("std");
const Date = @import("../Date.zig");
const market = @import("../market.zig");
const cache = @import("store.zig");
const Dividend = @import("../models/dividend.zig").Dividend;
const Split = @import("../models/split.zig").Split;
/// One cached symbol, as the caller found it.
pub const Entry = struct {
@ -603,3 +622,138 @@ test "collect-style exclusion: an excluded symbol produces no finding at all" {
try testing.expectEqual(@as(usize, 0), without.orphans.len);
try testing.expectEqual(@as(usize, 0), without.stale.len);
}
// Adjustment-basis staleness
//
// The candle cache is append-only. `getCandles` tops up from
// `last_date + 1`, and the newly-appended bars arrive with
// `adj_close == close` because nothing has gone ex after them yet -
// while every previously-cached bar keeps the adjustment basis it was
// originally fetched with. When the next distribution goes ex, the bars
// behind it should be marked down by its factor and nothing does it, so
// every total return spanning that ex-date reads low by roughly the
// missed yield.
//
// `CandleMeta.adj_basis` records how current a series' basis is. These
// two functions turn that into a verdict, split the same way as
// `collect` / `scan`: one gathers from disk, one decides.
/// Newest corporate-action ex-date cached for a symbol, or null when no
/// dividends or splits are on disk.
///
/// Splits count as much as dividends here, and arguably more: raw
/// `close` is not split-adjusted (hence `split.cumulativeSplitRatio`),
/// so an unapplied 2:1 split leaves older `adj_close` values wrong by
/// 50% rather than by a quarter's yield.
///
/// Allocates only transiently - the returned `Date` borrows nothing.
pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, symbol: []const u8) ?Date {
var newest: ?Date = null;
// `CacheResult` has no deinit - the caller owns `data`. Dividends
// carry owned strings, so they need `freeSlice`; splits are
// pure-numeric.
if (store.read(allocator, Dividend, symbol, null, .any)) |r| {
defer Dividend.freeSlice(allocator, r.data);
for (r.data) |d| {
if (newest == null or newest.?.lessThan(d.ex_date)) newest = d.ex_date;
}
}
if (store.read(allocator, Split, symbol, null, .any)) |r| {
defer allocator.free(r.data);
for (r.data) |sp| {
if (newest == null or newest.?.lessThan(sp.date)) newest = sp.date;
}
}
return newest;
}
/// Does a series' `adj_close` predate a corporate action it should
/// already reflect?
///
/// - `adj_basis`: `CandleMeta.adj_basis` - the bar date through which
/// the cached `adj_close` values reflect corporate actions.
/// - `last_bar`: `CandleMeta.last_date` - the newest bar held.
/// - `newest_action`: from `newestCorporateAction`, or null for a
/// symbol with no cached dividends or splits.
///
/// The `last_bar` bound is what keeps this convergent. A distribution
/// announced with a future ex-date is not yet reflected in *any*
/// provider's adjustment series, so treating it as stale would refetch
/// on every pass and never settle.
///
/// Pure: three dates in, bool out.
pub fn adjustmentBasisStale(adj_basis: Date, last_bar: Date, newest_action: ?Date) bool {
const newest = newest_action orelse return false;
if (last_bar.lessThan(newest)) return false;
return adj_basis.lessThan(newest);
}
test "adjustmentBasisStale: no corporate action means nothing to restate" {
// A non-payer has no ex-date to exceed the basis, so it must never
// escalate - not even with the epoch sentinel a legacy cache parses to.
try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), Date.fromYmd(2026, 8, 14), null));
try testing.expect(!adjustmentBasisStale(Date.epoch, Date.fromYmd(2026, 8, 14), null));
}
test "adjustmentBasisStale: action behind the basis is stale" {
const last_bar = Date.fromYmd(2026, 8, 14);
const ex = Date.fromYmd(2026, 6, 18);
// Basis at the newest bar already covers the ex-date.
try testing.expect(!adjustmentBasisStale(last_bar, last_bar, ex));
// Basis exactly at the ex-date covers it too (not `lessThan`).
try testing.expect(!adjustmentBasisStale(ex, last_bar, ex));
// Basis behind the ex-date: the bars in between were never marked down.
try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), last_bar, ex));
// The legacy-cache case: sentinel basis on a dividend payer. This is
// the shape every pre-adj_basis cache lands in.
try testing.expect(adjustmentBasisStale(Date.epoch, last_bar, ex));
}
test "adjustmentBasisStale: a not-yet-ex action is not stale" {
// Ex-date beyond the newest bar we hold. No provider has applied it
// either, so escalating would never converge.
try testing.expect(!adjustmentBasisStale(
Date.fromYmd(2026, 5, 1),
Date.fromYmd(2026, 8, 14),
Date.fromYmd(2026, 9, 17),
));
// Boundary: an ex-date exactly on the newest bar IS covered.
try testing.expect(adjustmentBasisStale(
Date.fromYmd(2026, 5, 1),
Date.fromYmd(2026, 8, 14),
Date.fromYmd(2026, 8, 14),
));
}
test "newestCorporateAction: takes the max across dividends and splits" {
const io = testing.io;
const a = testing.allocator;
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a);
defer a.free(dir_path);
var store = cache.Store.init(io, a, dir_path);
// Nothing cached at all.
try testing.expect(newestCorporateAction(a, &store, "SMPL") == null);
var divs = [_]Dividend{
.{ .ex_date = Date.fromYmd(2026, 3, 20), .amount = 1.79 },
.{ .ex_date = Date.fromYmd(2026, 6, 18), .amount = 1.90 },
};
store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 6, 18)));
// A later split must win over the later dividend.
var splits = [_]Split{.{ .date = Date.fromYmd(2026, 7, 1), .numerator = 2, .denominator = 1 }};
store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits });
try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 7, 1)));
// Splits alone (different symbol) still resolve.
store.write(Split, "SMPLB", splits[0..], .{ .seconds = cache.Ttl.splits });
try testing.expect(newestCorporateAction(a, &store, "SMPLB").?.eql(Date.fromYmd(2026, 7, 1)));
}

410
src/cache/store.zig vendored
View file

@ -67,6 +67,17 @@ pub const Ttl = struct {
/// month.
pub const tickers_funds: i64 = 30 * s_per_day;
pub const tickers_companies: i64 = 30 * s_per_day;
/// How long a genuine Tiingo 404 suppresses further Tiingo
/// attempts for a symbol (see `CandleMeta.tiingo_retry_after_s`).
///
/// Long enough that a symbol Tiingo genuinely does not carry
/// costs about one wasted 404 per month, short enough that
/// coverage changes get picked up without operator action.
/// Call sites layer `jitter_pct` on top so a batch of symbols
/// demoted in the same window does not all re-probe on the same
/// day.
pub const tiingo_backoff: i64 = 30 * s_per_day;
};
/// Cache TTL specification with optional per-key expiration jitter.
@ -894,7 +905,17 @@ pub const Store = struct {
/// candle metadata, stamping its `#!expires=` with `expires_at_s`
/// (the caller computes the market-aware freshness boundary; see
/// `market.nextCandleExpiry`).
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
///
/// `attrs` supplies the provider-state fields; `last_close`,
/// `last_date` and `adj_basis` are derived from the newest candle
/// written.
///
/// Deriving `adj_basis` here is the whole point of routing full
/// refetches through this function: replacing the file means every
/// bar carries the provider's current adjustment basis, which by
/// construction reflects every corporate action up to the newest
/// bar. `appendCandles` deliberately cannot do this.
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, attrs: CandleMetaAttrs, expires_at_s: i64) void {
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
defer self.allocator.free(srf_data);
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
@ -906,7 +927,14 @@ pub const Store = struct {
if (candles.len > 0) {
const last = candles[candles.len - 1];
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
self.updateCandleMeta(symbol, .{
.last_close = last.close,
.last_date = last.date,
.provider = attrs.provider,
.fail_count = attrs.fail_count,
.tiingo_retry_after_s = attrs.tiingo_retry_after_s,
.adj_basis = last.date,
}, expires_at_s);
}
}
@ -914,7 +942,17 @@ pub const Store = struct {
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
/// Also updates candle metadata, stamping its `#!expires=` with
/// `expires_at_s` (caller-computed market-aware boundary).
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
///
/// `meta` supplies every metadata field except `last_close` and
/// `last_date`, which are derived from the newest appended candle.
/// Callers are expected to pass the symbol's *existing* meta so
/// that fields describing the series as a whole survive an append
/// unchanged - appending bars does not re-derive anything about
/// the rows already on disk. `adj_basis` in particular MUST NOT
/// advance here: the appended bars do not restate the older rows'
/// `adj_close`, so claiming a newer basis would mask exactly the
/// staleness that field exists to detect.
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, meta: CandleMeta, expires_at_s: i64) void {
if (new_candles.len == 0) return;
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
@ -943,23 +981,25 @@ pub const Store = struct {
}
const last = new_candles[new_candles.len - 1];
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
var next = meta;
next.last_close = last.close;
next.last_date = last.date;
self.updateCandleMeta(symbol, next, expires_at_s);
}
/// Write (or refresh) candle metadata with a specific provider source.
/// Write (or refresh) candle metadata.
///
/// `expires_at_s` is the absolute Unix-seconds freshness boundary for
/// the `#!expires=` directive, computed by the caller via
/// `market.nextCandleExpiry` (close-anchored) or a short retry. The
/// cache layer no longer reads the wall clock for this - the boundary
/// is market-domain knowledge owned by the caller.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
const meta = CandleMeta{
.last_close = last_close,
.last_date = last_date,
.provider = provider,
.fail_count = fail_count,
};
///
/// Takes the whole `CandleMeta` rather than a field-per-parameter
/// list so that adding a metadata field does not churn every call
/// site. Callers that mean "same metadata, one field changed"
/// should copy the value they read and mutate the one field.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, meta: CandleMeta, expires_at_s: i64) void {
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| {
defer self.allocator.free(meta_data);
self.writeRaw(symbol, .candles_meta, meta_data) catch |err| {
@ -1348,27 +1388,118 @@ pub const Store = struct {
pub const CandleMeta = struct {
last_close: f64,
last_date: Date,
/// Which provider sourced the candle data. **No default
/// value on purpose** - SRF auto-elides fields whose value
/// equals their default, which would hide the provider line
/// when it equaled the implicit default. We want every cache
/// file to record its provider explicitly so cache inspection
/// can always answer "where did this come from?". Construction
/// sites must pass the provider explicitly.
/// Which provider sourced the candle data. Pure provenance -
/// this field answers "where did these bars come from?" and
/// nothing else. It deliberately does **not** drive provider
/// routing; see `tiingo_retry_after_s` for that. Conflating
/// the two is what produced the one-way drift to Yahoo that
/// this field's value used to cause.
///
/// **No default value on purpose** - SRF auto-elides fields
/// whose value equals their default, which would hide the
/// provider line when it equaled the implicit default. We
/// want every cache file to record its provider explicitly
/// so cache inspection can always answer "where did this
/// come from?". Construction sites must pass the provider
/// explicitly.
///
/// Cache compatibility: pre-2026-05 caches that elided the
/// provider field will fail to deserialize after this change
/// (SRF returns FieldNotFoundOnFieldWithoutDefaultValue).
/// `readCandleMeta` swallows the error and returns null,
/// making the symbol look like a cache miss - `getCandles`
/// then triggers a fresh fetch via `populateAllFromTiingo`,
/// then triggers a fresh fetch via `refetchFullHistory`,
/// which writes a new meta file with the provider explicit.
/// The wipe happens naturally on first use post-upgrade.
///
/// This is also why no *other* field on this struct may be
/// default-less: that cold-start path is the destructive one.
/// It writes a negative-cache marker over `candles_daily.srf`
/// when no provider carries the symbol, so routing a
/// populated cache through it risks discarding real history.
/// Fields added since (`tiingo_retry_after_s`, `adj_basis`)
/// are defaulted so legacy caches keep parsing.
provider: CandleProvider,
/// Consecutive transient failure count for the primary provider (Tiingo).
/// Incremented on ServerError; reset to 0 on success. When >= 3, the
/// symbol is degraded to a fallback provider until Tiingo recovers.
/// Consecutive transient-failure count for a candle fetch
/// (ServerError / connection failure). Reset to 0 on any
/// successful fetch. At >= 3, `getCandles` serves the stale
/// cached series rather than returning an error, so a provider
/// outage degrades to "slightly old prices" instead of "no
/// prices".
///
/// It does **not** switch providers. Provider selection is
/// `tiingo_retry_after_s`'s job, and only a genuine 404 moves
/// it - a transient outage says nothing about coverage.
fail_count: u8 = 0,
/// Unix-seconds instant before which Tiingo should not be
/// consulted for this symbol. `0` (the default) means "no
/// backoff - always try Tiingo first".
///
/// Set **only** when Tiingo returns a genuine 404, which is a
/// fact about coverage rather than about the request. A 400 /
/// 402 / malformed body falls back to Yahoo for that one call
/// but leaves this field alone, so the next call retries
/// Tiingo. Cleared back to `0` the moment Tiingo serves the
/// symbol again.
///
/// Why this exists: routing used to key off `provider ==
/// .yahoo`, which a single non-transient Tiingo failure would
/// latch permanently - Yahoo was then tried first, succeeded,
/// and rewrote `provider = .yahoo`, so Tiingo was never
/// consulted again. An explicit, expiring signal makes the
/// demotion recoverable and records *why* it happened.
///
/// Defaulted so legacy caches (which lack the field) parse
/// cleanly and simply behave as "no backoff".
tiingo_retry_after_s: i64 = 0,
/// The bar date through which this series' `adj_close` values
/// reflect corporate actions - i.e. the newest bar present at
/// the last *full* fetch.
///
/// Providers compute `adj_close` by scaling raw `close` by the
/// product of the adjustment factors for every distribution
/// *after* that bar. So the whole series' adjustment basis is
/// only as current as the fetch that produced it: a full fetch
/// as of date D reflects every ex-date <= D, and nothing later.
///
/// Appends never advance this. `getCandles` tops the cache up
/// incrementally, and the newly-appended bars arrive with
/// `adj_close == close` (nothing has gone ex after them yet)
/// while every previously-cached bar keeps its original basis.
/// When a distribution later goes ex, the bars behind it should
/// be marked down and nothing does it - so total returns read
/// low by roughly the missed yield. Comparing this field
/// against the newest known dividend/split ex-date is how
/// `getCandles` detects that and escalates to a full refetch.
///
/// `Date.epoch` (the default) means "unknown - assume stale".
/// Any real ex-date is later than it, so a dividend payer
/// escalates on its first stale pass; a symbol with no
/// corporate actions has no ex-date to exceed it and never
/// escalates. Defaulted rather than required precisely so
/// legacy caches keep parsing: making it required would route
/// every cached symbol through the cold-start path, which is
/// Tiingo-only and writes a negative-cache marker *over* the
/// candle file on a 404 - destroying history for any symbol
/// Tiingo does not carry.
adj_basis: Date = Date.epoch,
};
/// The subset of `CandleMeta` that describes provider state rather
/// than the candle data itself.
///
/// `cacheCandles` takes this instead of a whole `CandleMeta`
/// because a full fetch may be a cold start with no prior metadata
/// to copy - there is no honest `last_close` / `last_date` for the
/// caller to supply, and those get derived from the candles being
/// written anyway. `appendCandles` by contrast always has the
/// existing meta in hand (it read `last_date` to decide what to
/// fetch), so it takes the full struct and preserves everything it
/// does not derive.
pub const CandleMetaAttrs = struct {
provider: CandleProvider,
fail_count: u8 = 0,
tiingo_retry_after_s: i64 = 0,
};
pub const CandleProvider = enum {
@ -1376,17 +1507,22 @@ pub const Store = struct {
/// writes produce this value (TwelveData was demoted in an
/// earlier change because its `adj_close` was unreliable).
/// Cache reads still recognize the value for backwards
/// compatibility.
/// compatibility, and `getCandles` treats such a cache as
/// unusable so the symbol gets refetched.
twelvedata,
/// Legacy: candles were sourced from Yahoo Finance. No new
/// writes produce this value (Yahoo was removed from the
/// candle pipeline in the 2026-05 audit; Yahoo is still used
/// for `getQuote` real-time prices but not for historical
/// candles). Cache reads still recognize the value for
/// backwards compatibility.
/// Candles were sourced from Yahoo Finance.
///
/// Actively written. `fetchCandlesFromProviders` falls back to
/// Yahoo whenever Tiingo cannot serve a symbol, and
/// `refetchFullHistory` does the same for a full restatement.
/// Yahoo's `adj_close` is split- and dividend-adjusted like
/// Tiingo's, but its parser yields 0 for a JSON `null` element,
/// which the analytics layer then has to discard - so Tiingo is
/// preferred where both will answer.
yahoo,
/// Active: candles sourced from Tiingo. The only value
/// produced by current writes.
/// Candles were sourced from Tiingo. Preferred: it serves
/// dividends and splits from the same response, and its
/// `adj_close` is what the analytics layer is written against.
tiingo,
pub fn fromString(s: []const u8) CandleProvider {
@ -3568,7 +3704,7 @@ test "deserializeCandleMeta fails on old cache that elided provider field" {
// (model has no default for provider). The graceful handling is
// upstream: `readCandleMeta` swallows the deserialization error
// and returns null, which makes `getCandles` treat it as a cache
// miss and trigger a fresh fetch via `populateAllFromTiingo`.
// miss and trigger a fresh fetch via `refetchFullHistory`.
// The new fetch writes a meta file with the provider explicit.
//
// This test documents the failure mode and confirms it's not a
@ -3585,6 +3721,214 @@ test "deserializeCandleMeta fails on old cache that elided provider field" {
try std.testing.expectError(error.InvalidData, result);
}
test "CandleMeta.tiingo_retry_after_s defaults to 0 and is elided when unset" {
// The field is defaulted precisely so caches written before it
// existed keep parsing. SRF elides default-valued fields, so an
// unset backoff costs no bytes and reads back as "no backoff".
const allocator = std.testing.allocator;
const meta = Store.CandleMeta{
.last_close = 100.0,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .tiingo,
};
try std.testing.expectEqual(@as(i64, 0), meta.tiingo_retry_after_s);
const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 });
defer allocator.free(data);
try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s") == null);
}
test "CandleMeta.tiingo_retry_after_s round-trips when armed" {
const allocator = std.testing.allocator;
const meta = Store.CandleMeta{
.last_close = 100.0,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .yahoo,
.tiingo_retry_after_s = 1_790_000_000,
};
const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 });
defer allocator.free(data);
try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s:num:1790000000") != null);
const parsed = try Store.deserializeCandleMeta(allocator, data);
try std.testing.expectEqual(@as(i64, 1_790_000_000), parsed.tiingo_retry_after_s);
try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider);
}
test "legacy candles_meta without tiingo_retry_after_s still parses" {
// The whole migration story for this field: unlike `provider`
// (which is deliberately default-less and therefore wipes old
// caches), a defaulted field must NOT break existing caches. If
// this ever regresses, every cached symbol takes a full re-fetch
// on first use - and for symbols Tiingo does not carry, the
// cold-start path would write a negative-cache marker over a
// perfectly good candle history.
const allocator = std.testing.allocator;
const legacy =
\\#!srfv1
\\#!expires=1787000100
\\#!created=1786748010
\\last_close:num:776.34,last_date::2026-08-14,provider::yahoo
\\
;
const parsed = try Store.deserializeCandleMeta(allocator, legacy);
try std.testing.expectEqual(@as(i64, 0), parsed.tiingo_retry_after_s);
try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider);
try std.testing.expect(parsed.last_date.eql(Date.fromYmd(2026, 8, 14)));
}
test "appendCandles preserves caller-supplied provider state" {
// appendCandles derives only last_close/last_date; every other
// metadata field must pass through untouched. Commit-2's
// `adj_basis` depends on this exact property, so pin it now.
const io = std.testing.io;
const allocator = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
var store = Store.init(io, allocator, dir_path);
const seed = [_]Candle{.{
.date = Date.fromYmd(2026, 8, 13),
.open = 1,
.high = 1,
.low = 1,
.close = 1,
.adj_close = 1,
.volume = 1,
}};
store.cacheCandles("SMPL", seed[0..], .{ .provider = .yahoo, .tiingo_retry_after_s = 1_790_000_000 }, 9_999_999_999);
const before = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult;
try std.testing.expectEqual(@as(i64, 1_790_000_000), before.meta.tiingo_retry_after_s);
const more = [_]Candle{.{
.date = Date.fromYmd(2026, 8, 14),
.open = 2,
.high = 2,
.low = 2,
.close = 2,
.adj_close = 2,
.volume = 2,
}};
store.appendCandles("SMPL", more[0..], before.meta, 9_999_999_999);
const after = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult;
try std.testing.expectEqual(@as(i64, 1_790_000_000), after.meta.tiingo_retry_after_s);
try std.testing.expectEqual(Store.CandleProvider.yahoo, after.meta.provider);
// ...while the derived fields did advance.
try std.testing.expect(after.meta.last_date.eql(Date.fromYmd(2026, 8, 14)));
try std.testing.expectApproxEqAbs(@as(f64, 2), after.meta.last_close, 0.001);
}
// adj_basis: adjustment-basis staleness
//
// The cache is append-only. A distribution that goes ex after the last
// full fetch never marks down the bars behind it, so total returns read
// low by roughly the missed yield. `adj_basis` records how current the
// series' adjustment basis is; these tests pin the detection.
/// Build a one-bar-per-day series ending at `last`, for basis tests.
fn basisTestCandles(buf: []Candle, first: Date, count: usize) []Candle {
for (0..count) |i| {
const d = first.addDays(@intCast(i));
buf[i] = .{ .date = d, .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 };
}
return buf[0..count];
}
test "cacheCandles stamps adj_basis at the newest bar" {
// A full replace means every row carries the provider's current
// adjustment basis, which by construction covers every corporate
// action up to the newest bar.
const io = std.testing.io;
const allocator = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
var s = Store.init(io, allocator, dir_path);
var buf: [5]Candle = undefined;
const candles = basisTestCandles(&buf, Date.fromYmd(2026, 8, 10), 5);
s.cacheCandles("SMPL", candles, .{ .provider = .tiingo }, 9_999_999_999);
const meta = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
try std.testing.expect(meta.adj_basis.eql(Date.fromYmd(2026, 8, 14)));
try std.testing.expect(meta.last_date.eql(Date.fromYmd(2026, 8, 14)));
}
test "appendCandles does not advance adj_basis" {
// The core invariant. Appended bars arrive with adj_close == close
// (nothing has gone ex after them yet) and do NOT restate the rows
// already on disk, so claiming a newer basis would mask exactly the
// staleness this field exists to detect.
const io = std.testing.io;
const allocator = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
var s = Store.init(io, allocator, dir_path);
var buf: [3]Candle = undefined;
s.cacheCandles("SMPL", basisTestCandles(&buf, Date.fromYmd(2026, 6, 1), 3), .{ .provider = .tiingo }, 9_999_999_999);
const before = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
try std.testing.expect(before.adj_basis.eql(Date.fromYmd(2026, 6, 3)));
var more: [3]Candle = undefined;
s.appendCandles("SMPL", basisTestCandles(&more, Date.fromYmd(2026, 8, 12), 3), before, 9_999_999_999);
const after = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
// last_date advanced...
try std.testing.expect(after.last_date.eql(Date.fromYmd(2026, 8, 14)));
// ...but the basis is pinned to the last full fetch.
try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 3)));
}
test "legacy candles_meta without adj_basis parses to the epoch sentinel" {
// Same migration story as tiingo_retry_after_s, and the stakes are
// higher: making this field required would route every cached symbol
// through the cold-start path, which on a 404 writes a negative
// marker OVER candles_daily.srf and destroys the history.
const allocator = std.testing.allocator;
const legacy =
\\#!srfv1
\\#!expires=1787000100
\\#!created=1786748010
\\last_close:num:776.34,last_date::2026-08-14,provider::yahoo
\\
;
const parsed = try Store.deserializeCandleMeta(allocator, legacy);
try std.testing.expect(parsed.adj_basis.eql(Date.epoch));
// A sentinel basis on a dividend payer is exactly the "needs
// restatement" signal, so it must not be mistaken for "current".
try std.testing.expect(parsed.adj_basis.lessThan(Date.fromYmd(2026, 6, 18)));
}
test "adj_basis is elided when unset and round-trips when set" {
const allocator = std.testing.allocator;
const unset = Store.CandleMeta{
.last_close = 100.0,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .tiingo,
};
const bare = try Store.serializeCandleMeta(std.testing.io, allocator, unset, .{ .expires = 1 });
defer allocator.free(bare);
try std.testing.expect(std.mem.indexOf(u8, bare, "adj_basis") == null);
var set = unset;
set.adj_basis = Date.fromYmd(2026, 8, 14);
const full = try Store.serializeCandleMeta(std.testing.io, allocator, set, .{ .expires = 1 });
defer allocator.free(full);
try std.testing.expect(std.mem.indexOf(u8, full, "adj_basis::2026-08-14") != null);
const parsed = try Store.deserializeCandleMeta(allocator, full);
try std.testing.expect(parsed.adj_basis.eql(Date.fromYmd(2026, 8, 14)));
}
// writeRaw / appendRaw atomicity
//
// A concurrent reader hitting a cache file mid-write must never see a

View file

@ -239,7 +239,36 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
try out.print(", {s}", .{if (obs.local_fresh) "TTL still in the future" else "TTL lapsed"});
try out.print(", {s}", .{@tagName(m.meta.provider)});
if (m.meta.fail_count > 0) try out.print(", {d} consecutive failures", .{m.meta.fail_count});
// A live Tiingo backoff means this symbol is being served by
// Yahoo on purpose (Tiingo 404'd it), not by accident. Surface
// it so a provider that looks "wrong" can be explained.
if (m.meta.tiingo_retry_after_s > ctx.now_s) {
try out.print(", Tiingo backoff until {f}", .{Date.fromEpoch(m.meta.tiingo_retry_after_s)});
} else if (m.meta.tiingo_retry_after_s != 0) {
try out.print(", Tiingo backoff lapsed (retries next refresh)", .{});
}
try out.print("\n", .{});
// Adjustment basis. The cache is append-only, so a distribution
// that goes ex after the last full fetch never marks down the
// bars behind it - total returns then read low by roughly the
// missed yield. Report it here because the symptom (a slightly
// low 1Y total return) is otherwise invisible.
if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| {
if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) {
try out.print(
"adj basis {f} - STALE, {f} went ex behind it; total returns read low until restated\n",
.{ m.meta.adj_basis, newest_action },
);
} else {
try out.print(
"adj basis {f} - current through the newest corporate action ({f})\n",
.{ m.meta.adj_basis, newest_action },
);
}
} else {
try out.print("adj basis {f} - no dividends or splits cached, nothing to restate\n", .{m.meta.adj_basis});
}
} else {
try out.print("local nothing cached\n", .{});
}

View file

@ -83,20 +83,21 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
try out.print(")\nLatest close: {f}\n", .{Money.from(c[c.len - 1].close)});
// `dividends != null` indicates we got explicit dividend records
// from the provider. When false we still display total return
// (synthesized from adj_close, which most providers bake dividends
// into), but we surface a hint that explicit dividend data is
// missing.
// from the provider. When false we still display total return -
// `performance.totalReturns` degrades to the provider's `adj_close`,
// which is dividend-adjusted - but we surface a hint, because that
// series is only as current as the last full candle fetch (see
// `CandleMeta.adj_basis`) and understates by the missed yield when a
// distribution has gone ex since.
const has_explicit_divs = result.dividends != null;
const has_total = result.asof_total != null;
// -- As-of-date returns --
try cli.printBold(out, color, "\nAs-of {f}:\n", .{end_date});
try printReturnsTable(out, result.asof_price, if (has_total) result.asof_total else null, color);
try printReturnsTable(out, result.asof_price, result.asof_total, color);
// -- Month-end returns --
try cli.printBold(out, color, "\nMonth-end ({f}):\n", .{month_end});
try printReturnsTable(out, result.me_price, if (has_total) result.me_total else null, color);
try printReturnsTable(out, result.me_price, result.me_total, color);
if (!has_explicit_divs) {
try cli.printFg(out, color, cli.CLR_MUTED, "\nSet POLYGON_API_KEY for total returns with dividend reinvestment.\n", .{});

View file

@ -220,12 +220,32 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
}
}
// Per-symbol splits. Fetched alongside dividends because the totals
// row's total-return index is built from raw `close`, which is not
// split-adjusted - dividends without splits would read a 2:1 split
// as a -50% month. Splits ride along in the same Tiingo candle
// response the dividend warm above already paid for.
var split_map = std.StringHashMap([]const zfin.Split).init(allocator);
defer {
var it = split_map.iterator();
while (it.next()) |entry| {
allocator.free(@constCast(entry.value_ptr.*));
}
split_map.deinit();
}
for (pf_data.summary.allocations) |a| {
if (svc.getCachedSplits(allocator, a.symbol)) |spl| {
try split_map.put(a.symbol, spl.data);
}
}
var view = try review_view.buildReview(
allocator,
io,
pf_data.summary,
&pf_data.candle_map,
&dividend_map,
&split_map,
portfolio,
cm,
acct_map_opt,

View file

@ -34,7 +34,6 @@ const builtin = @import("builtin");
const srf = @import("srf");
const snapshot = @import("models/snapshot.zig");
const Date = @import("Date.zig");
const Candle = @import("models/candle.zig").Candle;
const timeline = @import("analytics/timeline.zig");
const valuation = @import("analytics/valuation.zig");
const imported_values = @import("data/imported_values.zig");
@ -564,35 +563,6 @@ pub fn resolveSnapshotDate(
// Pure-domain aggregation
/// Return the prefix of `candles` whose dates are `<= as_of`.
///
/// When `as_of` is null, returns the full slice unchanged (live mode
/// pass-through). When set, binary-searches for the first index
/// strictly after `as_of` and slices up to it. Zero-length slice
/// when `as_of` precedes all cached candles.
///
/// Candles are assumed sorted by date ascending. Used to truncate
/// benchmark and per-symbol price history for historical projections -
/// `performance.trailingReturns` uses the last candle's date as the
/// endpoint, so trimming the tail is equivalent to "compute as of
/// that date".
pub fn sliceCandlesAsOf(candles: []const Candle, as_of: ?Date) []const Candle {
const d = as_of orelse return candles;
if (candles.len == 0) return candles;
var lo: usize = 0;
var hi: usize = candles.len;
while (lo < hi) {
const mid = lo + (hi - lo) / 2;
const cd = candles[mid].date;
if (cd.lessThan(d) or cd.eql(d)) {
lo = mid + 1;
} else {
hi = mid;
}
}
return candles[0..lo];
}
/// Find the `scope=="liquid"` total in a snapshot. Returns 0.0 if not
/// present (old snapshots from before the liquid/illiquid split -
/// shouldn't happen in practice).
@ -1344,83 +1314,6 @@ test "aggregateSnapshotAllocations: aggregates by `symbol` (pricing), not `lot_s
try testing.expectApproxEqAbs(@as(f64, 6_750), sa.allocations[0].market_value, 0.01);
}
// sliceCandlesAsOf tests
fn makeTestCandle(y: i16, m: u8, d: u8, close: f64) Candle {
return .{
.date = Date.fromYmd(y, m, d),
.open = close,
.high = close,
.low = close,
.close = close,
.adj_close = close,
.volume = 0,
};
}
test "sliceCandlesAsOf: null as_of returns everything" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
};
const sliced = sliceCandlesAsOf(&candles, null);
try testing.expectEqual(@as(usize, 2), sliced.len);
}
test "sliceCandlesAsOf: empty input" {
const candles = [_]Candle{};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 1));
try testing.expectEqual(@as(usize, 0), sliced.len);
}
test "sliceCandlesAsOf: empty input with null as_of" {
const candles = [_]Candle{};
const sliced = sliceCandlesAsOf(&candles, null);
try testing.expectEqual(@as(usize, 0), sliced.len);
}
test "sliceCandlesAsOf: exact date match included" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
makeTestCandle(2024, 1, 3, 102),
makeTestCandle(2024, 1, 4, 103),
};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
try testing.expectEqual(@as(usize, 2), sliced.len);
try testing.expectApproxEqAbs(@as(f64, 101), sliced[sliced.len - 1].close, 0.001);
}
test "sliceCandlesAsOf: no exact match snaps to earlier" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 3, 102), // gap - no candle on the 2nd
makeTestCandle(2024, 1, 4, 103),
};
// Asking for Jan 2 returns everything through Jan 1 (nothing at/after Jan 2).
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
try testing.expectEqual(@as(usize, 1), sliced.len);
try testing.expectApproxEqAbs(@as(f64, 100), sliced[0].close, 0.001);
}
test "sliceCandlesAsOf: as_of before all candles returns empty" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2023, 12, 31));
try testing.expectEqual(@as(usize, 0), sliced.len);
}
test "sliceCandlesAsOf: as_of after all candles returns everything" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2026, 1, 1));
try testing.expectEqual(@as(usize, 2), sliced.len);
}
// resolveSnapshotDate tests
test "resolveSnapshotDate: exact match returns exact=true" {

View file

@ -1,3 +1,5 @@
const std = @import("std");
const testing = std.testing;
const Date = @import("../Date.zig");
/// A single OHLCV bar, normalized from any provider.
@ -21,13 +23,47 @@ pub const Candle = struct {
/// older cached candles or providers that don't populate
/// `adj_close` leave it at 0; in that case the raw close is
/// the best we can do.
///
/// The zero fallback matters beyond charts: Yahoo's parser yields
/// 0 for a JSON `null` element, and a risk series that divides by
/// it drops the month entirely. `analytics/portfolio_risk.zig`
/// reuses this for that reason - "adjusted close, or raw close
/// when the provider left it unusable" is the same question there.
pub fn chartClose(self: Candle) f64 {
return if (self.adj_close != 0) self.adj_close else self.close;
}
};
/// Return the prefix of `candles` whose dates are `<= as_of`.
///
/// When `as_of` is null, returns the full slice unchanged (live mode
/// pass-through). When set, binary-searches for the first index
/// strictly after `as_of` and slices up to it. Zero-length slice
/// when `as_of` precedes all cached candles.
///
/// Candles are assumed sorted by date ascending. Truncating the tail is
/// how "compute as of date D" is expressed throughout the analytics
/// layer: every window that derives its endpoint from the newest bar
/// (trailing returns, week change, month-end snaps) then lands on D
/// without needing its own as-of parameter.
pub fn sliceCandlesAsOf(candles: []const Candle, as_of: ?Date) []const Candle {
const d = as_of orelse return candles;
if (candles.len == 0) return candles;
var lo: usize = 0;
var hi: usize = candles.len;
while (lo < hi) {
const mid = lo + (hi - lo) / 2;
const cd = candles[mid].date;
if (cd.lessThan(d) or cd.eql(d)) {
lo = mid + 1;
} else {
hi = mid;
}
}
return candles[0..lo];
}
test "chartClose prefers adj_close when populated" {
const std = @import("std");
const c = Candle{
.date = Date.fromYmd(2024, 3, 7),
.open = 700.0,
@ -41,7 +77,6 @@ test "chartClose prefers adj_close when populated" {
}
test "chartClose falls back to close when adj_close is zero" {
const std = @import("std");
const c = Candle{
.date = Date.fromYmd(2024, 1, 1),
.open = 100.0,
@ -55,7 +90,6 @@ test "chartClose falls back to close when adj_close is zero" {
}
test "chartClose synthetic split has continuous chart values" {
const std = @import("std");
// Pre-split: close=300, adj_close=100 (after a 3:1 split)
// Post-split: close=100, adj_close=100
// Chart should see [100, 100] not [300, 100].
@ -79,3 +113,80 @@ test "chartClose synthetic split has continuous chart values" {
};
try std.testing.expectEqual(pre.chartClose(), post.chartClose());
}
// sliceCandlesAsOf tests
fn makeTestCandle(y: i16, m: u8, d: u8, close: f64) Candle {
return .{
.date = Date.fromYmd(y, m, d),
.open = close,
.high = close,
.low = close,
.close = close,
.adj_close = close,
.volume = 0,
};
}
test "sliceCandlesAsOf: null as_of returns everything" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
};
const sliced = sliceCandlesAsOf(&candles, null);
try testing.expectEqual(@as(usize, 2), sliced.len);
}
test "sliceCandlesAsOf: empty input" {
const candles = [_]Candle{};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 1));
try testing.expectEqual(@as(usize, 0), sliced.len);
}
test "sliceCandlesAsOf: empty input with null as_of" {
const candles = [_]Candle{};
const sliced = sliceCandlesAsOf(&candles, null);
try testing.expectEqual(@as(usize, 0), sliced.len);
}
test "sliceCandlesAsOf: exact date match included" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
makeTestCandle(2024, 1, 3, 102),
makeTestCandle(2024, 1, 4, 103),
};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
try testing.expectEqual(@as(usize, 2), sliced.len);
try testing.expectApproxEqAbs(@as(f64, 101), sliced[sliced.len - 1].close, 0.001);
}
test "sliceCandlesAsOf: no exact match snaps to earlier" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 3, 102), // gap - no candle on the 2nd
makeTestCandle(2024, 1, 4, 103),
};
// Asking for Jan 2 returns everything through Jan 1 (nothing at/after Jan 2).
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
try testing.expectEqual(@as(usize, 1), sliced.len);
try testing.expectApproxEqAbs(@as(f64, 100), sliced[0].close, 0.001);
}
test "sliceCandlesAsOf: as_of before all candles returns empty" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2023, 12, 31));
try testing.expectEqual(@as(usize, 0), sliced.len);
}
test "sliceCandlesAsOf: as_of after all candles returns everything" {
const candles = [_]Candle{
makeTestCandle(2024, 1, 1, 100),
makeTestCandle(2024, 1, 2, 101),
};
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2026, 1, 1));
try testing.expectEqual(@as(usize, 2), sliced.len);
}

View file

@ -250,7 +250,7 @@ pub const Client = struct {
}.f;
const uri = std.Uri.parse(url) catch |err| {
log.warn("http {s}: stage=uri_parse err={s} url={s}", .{ @tagName(method), @errorName(err), url });
log.warn("http {s}: stage=uri_parse err={s} url={f}", .{ @tagName(method), @errorName(err), redactUrl(url) });
return err;
};
const ms_uri_parse = stageElapsedMs(&t_stage, self.io);
@ -284,7 +284,7 @@ pub const Client = struct {
// TLS handshake. Logging at warn level (rather than debug)
// because DNS / connectivity failures are exactly what
// operators need to see immediately.
log.warn("http {s}: stage=connect err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=connect err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
defer req.deinit();
@ -294,24 +294,24 @@ pub const Client = struct {
var send_buf: [4096]u8 = undefined;
req.transfer_encoding = .{ .content_length = payload.len };
var bw = req.sendBodyUnflushed(&send_buf) catch |err| {
log.warn("http {s}: stage=send_body_open err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=send_body_open err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
bw.writer.writeAll(payload) catch |err| {
log.warn("http {s}: stage=send_body_write err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=send_body_write err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
bw.end() catch |err| {
log.warn("http {s}: stage=send_body_end err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=send_body_end err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
req.connection.?.flush() catch |err| {
log.warn("http {s}: stage=send_body_flush err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=send_body_flush err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
} else {
req.sendBodiless() catch |err| {
log.warn("http {s}: stage=send_bodiless err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=send_bodiless err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
}
@ -320,7 +320,7 @@ pub const Client = struct {
// Matches the default redirect capacity in std.http.Client.fetch.
var redirect_buffer: [8 * 1024]u8 = undefined;
var response = req.receiveHead(&redirect_buffer) catch |err| {
log.warn("http {s}: stage=receive_head err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=receive_head err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
const ms_receive_head = stageElapsedMs(&t_stage, self.io);
@ -359,7 +359,7 @@ pub const Client = struct {
var decompress_buffer: [64 * 1024]u8 = undefined;
const reader = response.readerDecompressing(&transfer_buffer, &decompress, &decompress_buffer);
_ = reader.streamRemaining(&aw.writer) catch |err| {
log.warn("http {s}: stage=stream_body err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
log.warn("http {s}: stage=stream_body err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
return err;
};
const ms_body = stageElapsedMs(&t_stage, self.io);
@ -368,7 +368,7 @@ pub const Client = struct {
const total_ms = @divTrunc(std.Io.Timestamp.now(self.io, .awake).nanoseconds - t_start, std.time.ns_per_ms);
log.debug(
"http {s}: ok status={d} bytes={d} total_ms={d} (uri_parse={d} connect={d} send={d} receive_head={d} body={d}) url={s}",
"http {s}: ok status={d} bytes={d} total_ms={d} (uri_parse={d} connect={d} send={d} receive_head={d} body={d}) url={f}",
.{
@tagName(method),
@intFromEnum(response.head.status),
@ -379,7 +379,7 @@ pub const Client = struct {
ms_send,
ms_receive_head,
ms_body,
url,
redactUrl(url),
},
);
@ -443,6 +443,117 @@ pub const Client = struct {
}
};
// URL redaction for logs
//
// Providers authenticate by query parameter, so a raw request URL is a
// credential. Every log line here used to print `url={s}` verbatim -
// including at `warn` level, which reaches production (ReleaseSafe
// defaults to `.info`). A single upstream 5xx was therefore enough to
// write a live API key into a cron log. Log through `redactUrl` and
// `{f}` instead; never interpolate a URL with `{s}`.
/// Placeholder substituted for a credential value.
const redacted_marker = "REDACTED";
/// Query-parameter names whose values are credentials.
///
/// Covers every in-tree provider (`token` for Tiingo, `apikey` for FMP
/// and TwelveData, `apiKey` for Polygon) plus the common variants a
/// future provider is likely to use. Matched case-insensitively.
///
/// **Adding a provider that authenticates by query parameter?** Add its
/// parameter name here. `looksLikeCredential` is a backstop for the
/// case where someone forgets, not a substitute for this list.
const credential_params = [_][]const u8{
"token",
"apikey",
"api_key",
"apitoken",
"api_token",
"key",
"access_token",
"auth",
"secret",
"password",
};
fn isCredentialParam(name: []const u8) bool {
for (credential_params) |candidate| {
if (std.ascii.eqlIgnoreCase(name, candidate)) return true;
}
return false;
}
/// Backstop heuristic: does this value look like an opaque secret?
///
/// True for a long run of credential-alphabet bytes and nothing else.
/// Every legitimate query value these providers send is either short
/// (symbols, intervals, booleans, CIKs, CUSIPs - all under 20 bytes) or
/// contains punctuation the alphabet below excludes (dates carry `-`
/// but are 10 bytes; symbol lists carry `,`). Real keys are 32-40 bytes
/// of hex or base62.
///
/// Deliberately biased toward over-redaction: a redacted debug line is
/// an inconvenience, a leaked key is an incident.
fn looksLikeCredential(value: []const u8) bool {
if (value.len < 20) return false;
for (value) |c| {
switch (c) {
'A'...'Z', 'a'...'z', '0'...'9', '_', '-' => {},
else => return false,
}
}
return true;
}
/// A URL wrapped for safe logging. Render with `{f}`.
pub const RedactedUrl = struct {
url: []const u8,
pub fn format(self: RedactedUrl, w: *std.Io.Writer) std.Io.Writer.Error!void {
// Everything up to '?' is scheme/host/path - no credentials.
const q = std.mem.indexOfScalar(u8, self.url, '?') orelse {
try w.writeAll(self.url);
return;
};
try w.writeAll(self.url[0 .. q + 1]);
// `buildUrl` percent-encodes '&' and '=' out of values, so
// splitting on them is unambiguous. Polygon hand-rolls its
// `apiKey=` suffix but uses the same delimiters.
var first = true;
var it = std.mem.splitScalar(u8, self.url[q + 1 ..], '&');
while (it.next()) |pair| {
if (!first) try w.writeByte('&');
first = false;
const eq = std.mem.indexOfScalar(u8, pair, '=') orelse {
try w.writeAll(pair);
continue;
};
const name = pair[0..eq];
const value = pair[eq + 1 ..];
try w.writeAll(name);
try w.writeByte('=');
if (isCredentialParam(name) or looksLikeCredential(value)) {
try w.writeAll(redacted_marker);
} else {
try w.writeAll(value);
}
}
}
};
/// Wrap a URL so `{f}` renders it with credential values elided.
///
/// Use this at every log site that reports a URL. The parameter name
/// and the non-secret values survive, so an operator can still see
/// which symbol and date range failed.
pub fn redactUrl(url: []const u8) RedactedUrl {
return .{ .url = url };
}
/// Build a URL with query parameters. Values are percent-encoded per RFC 3986.
pub fn buildUrl(
allocator: std.mem.Allocator,
@ -636,3 +747,105 @@ test "Response.verifyIntegrity: mismatched sha256 returns mismatch" {
else => try std.testing.expect(false),
}
}
// redactUrl
//
// Regression suite for a credential leak: every `url=` log line - nine
// of them at `warn`, which reaches production under ReleaseSafe - used
// to print the request URL verbatim, so any upstream failure wrote a
// live API key into the log.
/// Render through the `{f}` path the log statements use.
fn expectRedacted(expected: []const u8, url: []const u8) !void {
var buf: [512]u8 = undefined;
const got = try std.fmt.bufPrint(&buf, "{f}", .{redactUrl(url)});
try std.testing.expectEqualStrings(expected, got);
}
test "redactUrl: hides every in-tree provider's credential parameter" {
// Tiingo (`token`) - the key that actually leaked.
try expectRedacted(
"https://api.tiingo.com/tiingo/daily/SPY/prices?startDate=2026-08-07&endDate=2026-08-17&token=REDACTED",
"https://api.tiingo.com/tiingo/daily/SPY/prices?startDate=2026-08-07&endDate=2026-08-17&token=0123456789abcdef0123456789abcdef01234567",
);
// FMP and TwelveData (`apikey`).
try expectRedacted(
"https://financialmodelingprep.com/stable/earnings?symbol=SMPL&apikey=REDACTED",
"https://financialmodelingprep.com/stable/earnings?symbol=SMPL&apikey=cafebabecafebabecafebabecafebabe",
);
// Polygon (`apiKey`) - different case, and hand-rolled rather than
// built by `buildUrl`.
try expectRedacted(
"https://api.polygon.io/v3/reference/dividends?ticker=SMPL&apiKey=REDACTED",
"https://api.polygon.io/v3/reference/dividends?ticker=SMPL&apiKey=deadbeefdeadbeefdeadbeefdeadbeef",
);
}
test "redactUrl: keeps the diagnostically useful parts" {
// The point of these log lines is "which request failed?". Symbol,
// dates and interval must survive so they stay actionable.
try expectRedacted(
"https://api.example.com/v1/bars?symbol=SMPL&interval=1d&startDate=2026-08-07&adjusted=true&token=REDACTED",
"https://api.example.com/v1/bars?symbol=SMPL&interval=1d&startDate=2026-08-07&adjusted=true&token=0123456789abcdef0123456789abcdef01234567",
);
// A comma-separated symbol list is long but is not a credential.
try expectRedacted(
"https://api.example.com/quotes?symbols=SMPLA,SMPLB,SMPLC,SMPLD,SMPLE",
"https://api.example.com/quotes?symbols=SMPLA,SMPLB,SMPLC,SMPLD,SMPLE",
);
}
test "redactUrl: passes through URLs with no query string" {
try expectRedacted("https://zfin.example.org/SPY/diagnostics", "https://zfin.example.org/SPY/diagnostics");
try expectRedacted("https://zfin.example.org/SPY/candles?", "https://zfin.example.org/SPY/candles?");
}
test "redactUrl: tolerates malformed query segments" {
// A bare flag with no '=' must not be dropped or mangled.
try expectRedacted(
"https://api.example.com/x?flag&symbol=SMPL&token=REDACTED",
"https://api.example.com/x?flag&symbol=SMPL&token=0123456789abcdef0123456789abcdef01234567",
);
// An empty credential value stays redacted rather than revealing
// that the key was absent.
try expectRedacted("https://api.example.com/x?token=REDACTED", "https://api.example.com/x?token=");
}
test "redactUrl: heuristic backstops an unlisted parameter name" {
// If a future provider authenticates with a name nobody added to
// `credential_params`, an opaque 32-byte value still gets caught.
try expectRedacted(
"https://api.example.com/x?symbol=SMPL&sessionCredential=REDACTED",
"https://api.example.com/x?symbol=SMPL&sessionCredential=cafebabecafebabecafebabecafebabe",
);
}
test "looksLikeCredential: separates opaque secrets from real query values" {
// Secrets: 32-byte hex, 40-byte hex, base62-with-dash-and-underscore.
try std.testing.expect(looksLikeCredential("cafebabecafebabecafebabecafebabe"));
try std.testing.expect(looksLikeCredential("0123456789abcdef0123456789abcdef01234567"));
try std.testing.expect(looksLikeCredential("ya29_A0ARrdaM-abcdefghijklmnop"));
// Everything these providers legitimately send is either short...
try std.testing.expect(!looksLikeCredential("SMPL"));
try std.testing.expect(!looksLikeCredential("2026-08-07"));
try std.testing.expect(!looksLikeCredential("1d"));
try std.testing.expect(!looksLikeCredential("true"));
try std.testing.expect(!looksLikeCredential("0000320193"));
// ...or long but punctuated in ways a key never is.
try std.testing.expect(!looksLikeCredential("SMPLA,SMPLB,SMPLC,SMPLD,SMPLE"));
try std.testing.expect(!looksLikeCredential("2026-08-07T00:00:00.000Z"));
}
test "isCredentialParam: case-insensitive across provider spellings" {
try std.testing.expect(isCredentialParam("token"));
try std.testing.expect(isCredentialParam("apikey"));
try std.testing.expect(isCredentialParam("apiKey"));
try std.testing.expect(isCredentialParam("APIKEY"));
try std.testing.expect(isCredentialParam("api_key"));
try std.testing.expect(isCredentialParam("access_token"));
try std.testing.expect(!isCredentialParam("symbol"));
try std.testing.expect(!isCredentialParam("ticker"));
try std.testing.expect(!isCredentialParam("startDate"));
}

View file

@ -23,6 +23,7 @@ const Holding = @import("models/etf_profile.zig").Holding;
const SectorWeight = @import("models/etf_profile.zig").SectorWeight;
const Config = @import("Config.zig");
const cache = @import("cache/store.zig");
const freshness = @import("cache/freshness.zig");
const srf = @import("srf");
const analysis = @import("analytics/analysis.zig");
const transaction_log = @import("models/transaction_log.zig");
@ -133,6 +134,17 @@ pub fn isPermanentProviderFailure(err: anyerror) bool {
return err == error.NotFound;
}
/// Spread applied to `Ttl.tiingo_backoff` when a symbol is demoted off
/// Tiingo by a 404.
///
/// Policy lives here rather than in `TtlSpec` (see its doc comment).
/// 7% of 30 days is roughly +/-2 days, so a batch of symbols demoted in
/// the same window re-probes across five distinct days instead of all
/// on one. Sized against the cron cadence: with the refresh running
/// twice a day, five days of spread keeps the re-probe burst to a
/// handful of extra 404s per run.
const tiingo_backoff_jitter_pct: u8 = 7;
/// Result of a CUSIP-to-ticker lookup (provider-agnostic).
pub const CusipResult = OpenFigi.FigiResult;
@ -652,7 +664,7 @@ pub const DataService = struct {
// (market-aware next post-close / NAV-availability time).
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
const kind = market.classify(symbol);
s.cacheCandles(symbol, triple.candles, .tiingo, 0, expiryAfterFetch(now_s, kind, triple.candles));
s.cacheCandles(symbol, triple.candles, .{ .provider = .tiingo }, expiryAfterFetch(now_s, kind, triple.candles));
}
// Dividends and splits use the supplement write path: Tiingo's
// view merges into existing (typically Polygon-sourced) records
@ -666,6 +678,112 @@ pub const DataService = struct {
return triple;
}
/// Fixed start date for any full-history candle fetch. See
/// `populateAllFromTiingo`'s doc comment for the rationale.
const full_history_start: Date = Date.fromYmd(2000, 1, 1);
/// Replace a symbol's entire candle series, restating `adj_close`
/// from whichever provider will serve it.
///
/// This is the only way a cached series' adjustment basis ever
/// advances - see `CandleMeta.adj_basis`. Incremental appends
/// cannot restate the rows already on disk, so once a distribution
/// goes ex behind them the whole series reads low until something
/// rewrites it. That something is this function.
///
/// Tiingo first (it returns dividends and splits in the same
/// response, and its `adj_close` is the one the analytics layer is
/// written against), falling back to a full-range Yahoo fetch when
/// Tiingo does not carry the symbol.
///
/// **Never writes a negative-cache entry.** Callers reach this
/// holding a working series; `writeNegative` would overwrite
/// `candles_daily.srf` with a marker and destroy that history. A
/// restatement that cannot be completed is a "try again later", not
/// a verdict about the symbol. Callers should treat any error as
/// "keep the existing series and carry on".
///
/// Returns `error.NotFound` only when *every* provider says it does
/// not carry the symbol - that is the one outcome a cold-start
/// caller may legitimately turn into a negative-cache entry. Any
/// other failure surfaces as `FetchFailed` / `TransientError` /
/// `AuthError` so a network blip can never be mistaken for
/// "this symbol does not exist".
fn refetchFullHistory(
self: *DataService,
symbol: []const u8,
today: Date,
now_s: i64,
prefer_yahoo: bool,
) (DataError || error{NotFound})![]Candle {
self.assertNetworkAllowed("getCandles refetchFullHistory");
const kind = market.classify(symbol);
var s = self.store();
// Tracks whether each provider affirmatively said "no such
// symbol", as opposed to failing for some other reason. Only
// unanimous 404s justify the caller writing a negative entry.
var tiingo_not_found = false;
if (!prefer_yahoo) {
if (self.populateAllFromTiingo(symbol)) |triple| {
defer Dividend.freeSlice(self.allocator, triple.dividends);
defer self.allocator.free(triple.splits);
if (triple.candles.len > 0) return triple.candles;
// An empty full-history response is not a restatement;
// fall through rather than caching emptiness.
self.allocator.free(triple.candles);
log.warn("{s}: Tiingo full history returned no bars, trying Yahoo", .{symbol});
} else |err| {
// Transient failures must not silently degrade to a
// second provider - the caller needs to know to retry.
if (err == error.RateLimited or isTransientError(err)) return DataError.TransientError;
if (err == error.Unauthorized) {
log.err("{s}: Tiingo auth failed during restatement - check TIINGO_API_KEY", .{symbol});
return DataError.AuthError;
}
tiingo_not_found = isPermanentProviderFailure(err);
log.info("{s}: Tiingo cannot serve full history ({s}), trying Yahoo", .{ symbol, @errorName(err) });
}
}
// Yahoo fallback. `fetchCandles` takes an arbitrary range, so a
// full-history request is just a wide one. Yahoo carries no
// dividend/split endpoint here; those caches are Polygon-primary
// and merged separately, so leaving them untouched is correct.
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, full_history_start, today)) |candles| {
if (candles.len > 0) {
s.cacheCandles(symbol, candles, .{
.provider = .yahoo,
// Preserve the Tiingo verdict this call just
// learned, so the next pass doesn't re-probe.
.tiingo_retry_after_s = if (tiingo_not_found)
cache.computeExpires(
now_s,
.{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct },
symbol,
)
else
0,
}, expiryAfterFetch(now_s, kind, candles));
log.info("{s}: full history restated from Yahoo ({d} bars)", .{ symbol, candles.len });
return candles;
}
self.allocator.free(candles);
log.warn("{s}: Yahoo full history returned no bars", .{symbol});
} else |err| {
log.warn("{s}: Yahoo full history failed: {s}", .{ symbol, @errorName(err) });
// Both providers affirmatively disclaim the symbol.
if (tiingo_not_found and isPermanentProviderFailure(err)) return error.NotFound;
}
} else |_| {
log.warn("{s}: Yahoo provider not available for full history", .{symbol});
}
return DataError.FetchFailed;
}
/// Invalidate cached data for a symbol so the next get* call forces a fresh fetch.
pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void {
var s = self.store();
@ -678,6 +796,73 @@ pub const DataService = struct {
// Public data methods
/// What a candle fetch learned about Tiingo's coverage of a symbol.
///
/// This is deliberately three-valued. The old code collapsed
/// "Tiingo does not carry this symbol" and "Tiingo failed this
/// request" into a single permanent demotion to Yahoo, which is
/// how 22 of 32 cached symbols drifted off Tiingo. Only
/// `.not_found` is a statement about the symbol; everything else
/// is a statement about one HTTP call and must not be remembered.
pub const TiingoCoverage = enum {
/// Tiingo served the request - it definitely covers this symbol.
covered,
/// Tiingo returned a genuine 404 - it does not carry this symbol.
not_found,
/// Tiingo was not consulted, or failed for a reason that says
/// nothing about coverage (active backoff, no API key, 400,
/// 402, malformed body). Nothing new learned; leave any
/// existing backoff exactly as it was.
unknown,
};
/// Fold a fetch's outcome into a symbol's candle metadata.
///
/// Called on any *successful* candle fetch, so `fail_count` resets
/// to 0. The interesting part is `tiingo_retry_after_s`:
///
/// - `.covered` -> clear the backoff. Tiingo just served this
/// symbol, so any prior 404 is stale news. This
/// is what walks a previously-demoted symbol
/// back onto Tiingo.
/// - `.not_found` -> arm the backoff at `Ttl.tiingo_backoff` with
/// per-symbol jitter, so a batch demoted in the
/// same window does not all re-probe on one day.
/// - `.unknown` -> leave it untouched. Either Tiingo was never
/// asked, or it failed for a reason that says
/// nothing about coverage.
fn applyTiingoCoverage(
meta: cache.Store.CandleMeta,
symbol: []const u8,
now_s: i64,
provider: cache.Store.CandleProvider,
coverage: TiingoCoverage,
) cache.Store.CandleMeta {
var next = meta;
next.provider = provider;
next.fail_count = 0;
switch (coverage) {
.covered => {
if (meta.tiingo_retry_after_s != 0) {
log.info("{s}: provider converted {t} -> tiingo, clearing Tiingo backoff", .{ symbol, meta.provider });
}
next.tiingo_retry_after_s = 0;
},
.not_found => {
next.tiingo_retry_after_s = cache.computeExpires(
now_s,
.{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct },
symbol,
);
log.info("{s}: Tiingo NotFound, backing off until {f}", .{ symbol, Date.fromEpoch(next.tiingo_retry_after_s) });
},
.unknown => {},
}
return next;
}
/// Fetch candles from providers with error classification.
///
/// Error handling:
@ -685,32 +870,42 @@ pub const DataService = struct {
/// - NotFound/ParseError/InvalidResponse from Tiingo -> try Yahoo (symbol-level issue)
/// - Unauthorized -> TransientError (config problem, stop refresh)
///
/// The `preferred` param controls incremental fetch consistency: use the same
/// provider that sourced the existing cache data.
/// `prefer_yahoo` skips the Tiingo attempt entirely. Callers derive
/// it from `CandleMeta.tiingo_retry_after_s` - i.e. "Tiingo told us
/// 404 recently, don't waste the call". It is NOT derived from
/// which provider sourced the cache; see the `provider` field's
/// doc comment for why that distinction matters.
fn fetchCandlesFromProviders(
self: *DataService,
symbol: []const u8,
from: Date,
to: Date,
preferred: cache.Store.CandleProvider,
) (DataError || error{NotFound})!struct { candles: []Candle, provider: cache.Store.CandleProvider } {
// If preferred is Yahoo (degraded symbol), try Yahoo first
if (preferred == .yahoo) {
prefer_yahoo: bool,
) (DataError || error{NotFound})!struct {
candles: []Candle,
provider: cache.Store.CandleProvider,
tiingo_coverage: TiingoCoverage,
} {
// Under an active Tiingo backoff, go straight to Yahoo.
if (prefer_yahoo) {
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Yahoo (preferred)", .{symbol});
return .{ .candles = candles, .provider = .yahoo };
log.debug("{s}: candles from Yahoo (Tiingo backoff active)", .{symbol});
return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = .unknown };
} else |err| {
log.warn("{s}: Yahoo (preferred) failed: {s}", .{ symbol, @errorName(err) });
log.warn("{s}: Yahoo (Tiingo backoff active) failed: {s}", .{ symbol, @errorName(err) });
}
} else |_| {}
}
// Primary: Tiingo
// Primary: Tiingo. `coverage` accumulates what this attempt
// taught us, so the Yahoo fallback below can report it back
// to the caller without re-deriving it from the error.
var coverage: TiingoCoverage = .unknown;
if (self.getProvider(Tiingo)) |tg| {
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Tiingo", .{symbol});
return .{ .candles = candles, .provider = .tiingo };
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
} else |err| {
log.warn("{s}: Tiingo failed: {s}", .{ symbol, @errorName(err) });
@ -725,7 +920,7 @@ pub const DataService = struct {
self.rateLimitBackoff();
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Tiingo (after rate limit backoff)", .{symbol});
return .{ .candles = candles, .provider = .tiingo };
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
} else |retry_err| {
log.warn("{s}: Tiingo retry after backoff failed: {s}", .{ symbol, @errorName(retry_err) });
if (retry_err == error.RateLimited) {
@ -733,7 +928,7 @@ pub const DataService = struct {
self.rateLimitBackoff();
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Tiingo (after second backoff)", .{symbol});
return .{ .candles = candles, .provider = .tiingo };
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
} else |_| {}
}
// Exhausted rate limit retries - treat as transient
@ -746,19 +941,31 @@ pub const DataService = struct {
return DataError.TransientError;
}
// NotFound, ParseError, InvalidResponse - symbol-level issue, try Yahoo
log.info("{s}: Tiingo does not have this symbol, trying Yahoo", .{symbol});
// NotFound, ParseError, InvalidResponse - fall back to
// Yahoo for this call. Only a genuine 404 is a
// statement about Tiingo's coverage; a 400 / 402 /
// malformed body says something about this request and
// must not earn a remembered demotion. Mirrors the rule
// `isPermanentProviderFailure` already applies to the
// negative cache.
if (isPermanentProviderFailure(err)) {
coverage = .not_found;
log.info("{s}: Tiingo does not carry this symbol, trying Yahoo", .{symbol});
} else {
log.info("{s}: Tiingo request failed ({s}) - not a coverage verdict, trying Yahoo for this call only", .{ symbol, @errorName(err) });
}
}
} else |_| {
log.warn("{s}: Tiingo provider not available (no API key?)", .{symbol});
}
// Fallback: Yahoo (symbol not on Tiingo)
if (preferred != .yahoo) {
// Fallback: Yahoo. Skipped when we already tried Yahoo first
// (active backoff) and it failed - no point asking twice.
if (!prefer_yahoo) {
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.info("{s}: candles from Yahoo (Tiingo fallback)", .{symbol});
return .{ .candles = candles, .provider = .yahoo };
return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = coverage };
} else |err| {
log.warn("{s}: Yahoo fallback also failed: {s}", .{ symbol, @errorName(err) });
}
@ -932,7 +1139,17 @@ pub const DataService = struct {
// (Force-refresh skips server sync too: the user explicitly
// asked for fresh provider data.)
if (!opts.force_refresh and self.syncCandlesFromServer(symbol)) {
if (s.isCandleMetaFresh(symbol)) {
// Re-read meta: the sync wrote the server's bytes
// verbatim, so its view of both freshness AND
// adjustment basis is now ours. `serverBarRegression`
// only guards `last_date` going backwards - it cannot
// see a same-dated file whose historical adj_close is
// stale, so the basis has to be re-checked here.
const synced = if (s.readCandleMeta(symbol)) |sm| sm.meta else m;
const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol);
if (s.isCandleMetaFresh(symbol) and
!freshness.adjustmentBasisStale(synced.adj_basis, synced.last_date, synced_action))
{
log.debug("{s}: candles synced from server and fresh", .{symbol});
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
@ -947,6 +1164,36 @@ pub const DataService = struct {
// this stale path (next post-close / NAV-availability time).
const expires = market.nextCandleExpiry(now_s, kind);
// A corporate action has gone ex since the basis that
// produced this series, so the cached `adj_close` values
// behind it were never marked down. Appending cannot fix
// that - only replacing the file can. Do this BEFORE the
// incremental fetch so a symbol that needs both a top-up
// and a restatement costs one full fetch, not an append
// followed by a second pass.
//
// Deliberately not checked on the fresh-cache path above:
// the candle TTL lapses at least once per trading day, so
// detection is at most a day behind, and paying a
// dividend/split cache read on every getCandles call would
// tax the hot portfolio-pricing path for nothing.
if (freshness.adjustmentBasisStale(
m.adj_basis,
m.last_date,
freshness.newestCorporateAction(self.allocator, &s, symbol),
)) {
log.info("{s}: restating full history (adj_basis {f} predates a corporate action)", .{ symbol, m.adj_basis });
if (self.refetchFullHistory(symbol, today, now_s, now_s < m.tiingo_retry_after_s)) |candles| {
return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
} else |err| {
// Restatement is best-effort. The existing series
// is untouched and still usable, just understated
// by the missed adjustment - fall through to the
// normal top-up and retry on the next stale pass.
log.warn("{s}: full-history restatement failed ({s}); adjustment basis remains stale, falling back to incremental append", .{ symbol, @errorName(err) });
}
}
// Only skip the fetch when we already hold the latest
// *available* bar (weekend/holiday/pre-close gap, or
// caught up): just bump the TTL. Gating on the
@ -959,18 +1206,20 @@ pub const DataService = struct {
// boundary while the lag check reported it lagging (the
// Friday-17:00 deadlock that exited 75 every retry).
if (!market.shouldRefresh(now_s, kind, m.last_date)) {
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires);
s.updateCandleMeta(symbol, m, expires);
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
} else {
// Incremental fetch from day after last cached candle
self.assertNetworkAllowed("getCandles incremental fetchCandlesFromProviders");
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch |err| {
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, now_s < m.tiingo_retry_after_s) catch |err| {
if (err == DataError.TransientError) {
// Increment fail_count for this symbol
const new_fail_count = m.fail_count +| 1; // saturating add
log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s);
var degraded = m;
degraded.fail_count = new_fail_count;
s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s);
// If degraded (fail_count >= 3), return stale data rather than failing
if (new_fail_count >= 3) {
@ -987,6 +1236,10 @@ pub const DataService = struct {
};
const new_candles = result.candles;
// Fold what this fetch learned about Tiingo coverage
// into the metadata we're about to write.
const next_meta = applyTiingoCoverage(m, symbol, now_s, result.provider, result.tiingo_coverage);
if (new_candles.len == 0) {
// No new candles. Either a genuine non-trading-day
// gap (weekend/holiday), the provider hasn't posted
@ -999,7 +1252,7 @@ pub const DataService = struct {
// an un-modeled closure stops thrashing and waits
// for the next real session.
self.allocator.free(new_candles);
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date));
s.updateCandleMeta(symbol, next_meta, market.staleCandleExpiry(now_s, kind, m.last_date));
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
} else {
@ -1007,7 +1260,7 @@ pub const DataService = struct {
// TTL via `expiryAfterFetch`, NOT the precomputed
// next-boundary `expires`: getting a bar back does not
// mean getting the RIGHT bar back.
s.appendCandles(symbol, new_candles, result.provider, 0, expiryAfterFetch(now_s, kind, new_candles));
s.appendCandles(symbol, new_candles, next_meta, expiryAfterFetch(now_s, kind, new_candles));
if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
self.allocator.free(new_candles);
return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
@ -1034,45 +1287,50 @@ pub const DataService = struct {
log.debug("{s}: candles synced from server but stale, falling through to full fetch", .{symbol});
}
// No usable cache - full fetch via the orchestrated Tiingo
// helper, which writes candles + dividends + splits caches in
// one shot from a single HTTP response. The fixed start date
// (see `populateAllFromTiingo`) is 2000-01-01, deep enough to
// cover a 10Y trailing-return window even when `--as-of`
// back-dates the reference into 2014-era imported portfolio
// history, plus a buffer for older corporate actions like
// SPYM's 2017-10-16 split.
// No usable cache - full fetch. Tiingo first (it returns
// candles + dividends + splits from one response), falling back
// to a full-range Yahoo fetch when Tiingo does not carry the
// symbol. The fixed start date (see `populateAllFromTiingo`) is
// 2000-01-01, deep enough to cover a 10Y trailing-return window
// even when `--as-of` back-dates the reference into 2014-era
// imported portfolio history, plus a buffer for older corporate
// actions like SPYM's 2017-10-16 split.
//
// The Yahoo fallback matters here beyond convenience: this
// branch used to be Tiingo-only, so a symbol Tiingo does not
// carry could not be cold-started at all - it 404'd, wrote a
// negative-cache marker *over* candles_daily.srf, and stayed
// unavailable until `cache clear`. Any symbol that reached this
// branch with a populated cache would have had its history
// destroyed.
log.debug("{s}: fetching full candle history from provider", .{symbol});
self.assertNetworkAllowed("getCandles full populateAllFromTiingo");
const triple = self.populateAllFromTiingo(symbol) catch |err| {
if (err == error.RateLimited or err == error.ServerError or err == error.RequestFailed) {
const prior_backoff: i64 = if (meta_result) |mr| mr.meta.tiingo_retry_after_s else 0;
const candles = self.refetchFullHistory(symbol, today, now_s, now_s < prior_backoff) catch |err| {
if (err == DataError.TransientError) {
// Transient: increment fail_count on existing meta so
// we know to back off if this keeps happening.
if (meta_result) |mr| {
const new_fail_count = mr.meta.fail_count +| 1;
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s);
var degraded = mr.meta;
degraded.fail_count = mr.meta.fail_count +| 1;
s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s);
}
return DataError.TransientError;
}
// Only a genuine NotFound means "this symbol has no candle
// data on Tiingo" (the sole historical-candle provider since
// the 2026-05 audit) and earns a sticky negative-cache entry.
// Unauthorized / InvalidResponse / PaymentRequired and the
// like are not permanent facts about the symbol (transient
// auth misconfig, a malformed response) - fail this call but
// stay retryable, matching fetchCached's policy. This matters
// now that the candle negative cache is actually honored: a
// bad negative would otherwise stick until --refresh.
if (isPermanentProviderFailure(err)) s.writeNegative(symbol, .candles_daily);
// Only a unanimous NotFound - every provider affirmatively
// disclaims the symbol - earns a sticky negative-cache
// entry. Auth trouble, a malformed response, or a Yahoo
// network blip are not permanent facts about the symbol, so
// they fail this call but stay retryable. This matters
// because the negative marker replaces the candle file: a
// bad negative both suppresses retries until `--refresh`
// and throws away whatever history was cached.
if (err == error.NotFound) s.writeNegative(symbol, .candles_daily);
if (err == DataError.AuthError) return DataError.AuthError;
return DataError.FetchFailed;
};
// populateAllFromTiingo writes all three caches itself; we
// free the slices we don't return.
defer Dividend.freeSlice(self.allocator, triple.dividends);
defer self.allocator.free(triple.splits);
return .{ .data = triple.candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
}
/// Fetch dividend history for a symbol.
@ -1983,9 +2241,18 @@ pub const DataService = struct {
/// See `tmp/multi-ticker-audit.md` for the cross-validation evidence.
pub fn getTrailingReturns(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!struct {
asof_price: performance.TrailingReturns,
asof_total: ?performance.TrailingReturns,
/// Total return as of the newest bar. Non-optional: it is always
/// computable once candles exist, because `performance.totalReturns`
/// degrades to the adj_close series internally when no dividend
/// record is available. It was `?TrailingReturns`, and the
/// optionality invited consumers to write
/// `asof_total orelse asof_price` - silently publishing a
/// price-only return labelled as total return, understated by the
/// full dividend yield.
asof_total: performance.TrailingReturns,
me_price: performance.TrailingReturns,
me_total: ?performance.TrailingReturns,
/// Month-end total return. Non-optional for the same reason.
me_total: performance.TrailingReturns,
candles: []Candle,
dividends: ?[]Dividend,
source: Source,
@ -2014,34 +2281,17 @@ pub const DataService = struct {
// Month-end (end = last business day of prior month)
const me_price = performance.trailingReturnsPriceOnlyMonthEnd(c, splits, today);
// Total return: dividend-reinvested when dividends are
// available; otherwise fall back to adj_close-based total
// return (which captures dividends for providers like Tiingo
// that bake dividends into adj_close).
// Total return. `performance.totalReturns` owns the
// dividend-reinvestment vs adj_close merge, including the
// no-dividend-record fallback, so there is one code path here
// rather than a branch per availability case.
var divs: ?[]Dividend = null;
var asof_total: ?performance.TrailingReturns = null;
var me_total: ?performance.TrailingReturns = null;
// adj_close-based total return is the fallback. With Tiingo
// (the default provider) adj_close is already dividend-
// adjusted, so this gives a reasonable total-return estimate
// even when explicit dividend records are missing.
const asof_adj = performance.trailingReturns(c);
const me_adj = performance.trailingReturnsMonthEnd(c, today);
if (self.getDividends(symbol, opts)) |div_result| {
divs = div_result.data;
const asof_div = performance.trailingReturnsWithDividends(c, div_result.data);
const me_div = performance.trailingReturnsMonthEndWithDividends(c, div_result.data, today);
asof_total = performance.withDividendFallback(asof_div, asof_adj);
me_total = performance.withDividendFallback(me_div, me_adj);
} else |_| {
// No dividend data: still surface the adj_close-based
// total return rather than null, since Tiingo's
// adj_close already includes dividend adjustment.
asof_total = asof_adj;
me_total = me_adj;
}
} else |_| {}
const asof_total = performance.totalReturns(c, divs, today);
const me_total = performance.totalReturnsMonthEnd(c, divs, today);
return .{
.asof_price = asof_price,
@ -2133,6 +2383,18 @@ pub const DataService = struct {
return .{ .data = result.data, .source = .cached, .timestamp = result.timestamp, .allocator = allocator };
}
/// Cache-only split read, mirroring `getCachedDividends`.
///
/// Splits travel with dividends wherever raw `close` is being
/// adjusted: `close` is not split-adjusted, so a total-return series
/// built from dividends alone reads a 2:1 split as a -50% month.
/// Callers that fetch one should fetch both.
pub fn getCachedSplits(self: *DataService, allocator: std.mem.Allocator, symbol: []const u8) ?FetchResult(Split) {
var s = self.store();
const result = s.read(allocator, Split, symbol, null, .any) orelse return null;
return .{ .data = result.data, .source = .cached, .timestamp = result.timestamp, .allocator = allocator };
}
// Portfolio price loading
/// Status emitted for each symbol during price loading.
@ -3791,7 +4053,7 @@ test "getCandles offline mode returns cached data without network" {
.{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
.{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 },
};
store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
store.cacheCandles("TEST", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Set the test guard: any network call would panic. We expect
// the offline-mode path NOT to touch the network.
@ -3892,6 +4154,38 @@ test "loadAllDividends: honors skip_network for every symbol, and one miss does
try std.testing.expect(svc.getCachedDividends(allocator, "TSTA") == null);
}
test "getCachedSplits: reads cache only and reports absence" {
// Splits travel with dividends wherever raw `close` is adjusted, so
// this mirrors `getCachedDividends` exactly - including that an
// absent symbol returns null rather than an empty slice, which is a
// different fact (nothing cached vs. cached-and-never-split).
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
const config = Config{ .cache_dir = dir_path };
var svc = DataService.init(io, allocator, config);
defer svc.deinit();
var splits = [_]Split{
.{ .date = Date.fromYmd(2024, 6, 10), .numerator = 10, .denominator = 1 },
};
var store = svc.store();
store.write(Split, "TSTB", splits[0..], cache.DataType.splits.ttl());
svc.panic_on_network_attempt = true;
const b = svc.getCachedSplits(allocator, "TSTB") orelse return error.TestUnexpectedResult;
defer b.deinit();
try std.testing.expectEqual(@as(usize, 1), b.data.len);
try std.testing.expectApproxEqAbs(@as(f64, 10.0), b.data[0].ratio(), 1e-9);
try std.testing.expect(svc.getCachedSplits(allocator, "TSTA") == null);
}
test "loadAllDividends: empty symbol list is a no-op" {
const allocator = std.testing.allocator;
const io = std.testing.io;
@ -3946,7 +4240,7 @@ test "loadAllPrices offline mode skips network and returns cached" {
var fresh_candles = [_]Candle{
.{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
};
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
store.cacheCandles("FRESH", fresh_candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Symbol with no cache at all.
// (no setup needed - just passes a symbol that doesn't exist)
@ -3972,6 +4266,168 @@ test "loadAllPrices offline mode skips network and returns cached" {
try std.testing.expectEqual(@as(usize, 1), result.failed_count);
}
// adjustment-basis restatement
test "getCandles offline never escalates a stale adjustment basis" {
// Restatement does network I/O, so `skip_network` must return the
// cached series untouched rather than escalating. Pinned with
// `panic_on_network_attempt`, which fires inside
// `refetchFullHistory`'s `assertNetworkAllowed` if this regresses.
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
const config = Config{ .cache_dir = dir_path };
var svc = DataService.init(io, allocator, config);
defer svc.deinit();
var store = svc.store();
var candles = [_]Candle{
.{ .date = Date.fromYmd(2026, 8, 12), .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 },
.{ .date = Date.fromYmd(2026, 8, 13), .open = 11, .high = 11, .low = 11, .close = 11, .adj_close = 11, .volume = 1 },
};
store.cacheCandles("SMPL", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Roll the basis back behind a cached ex-date so the series is
// unambiguously due for restatement.
var divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 7, 1), .amount = 1.0 }};
store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
var meta = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta;
meta.adj_basis = Date.fromYmd(2026, 6, 1);
store.updateCandleMeta("SMPL", meta, 1); // expiry in the past => stale
try std.testing.expect(freshness.adjustmentBasisStale(
meta.adj_basis,
meta.last_date,
freshness.newestCorporateAction(allocator, &store, "SMPL"),
));
svc.panic_on_network_attempt = true;
const result = try svc.getCandles("SMPL", .{ .skip_network = true });
defer result.deinit();
// Served from cache, series intact, basis untouched.
try std.testing.expectEqual(@as(usize, 2), result.data.len);
const after = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta;
try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 1)));
// And critically, no negative-cache marker was written over the
// candle file.
try std.testing.expect(!store.isNegative("SMPL", .candles_daily));
}
// Tiingo coverage bookkeeping //
// Regression suite for the one-way drift to Yahoo. The old code
// routed on `CandleMeta.provider == .yahoo`, so any non-transient
// Tiingo failure latched permanently: Yahoo was tried first,
// succeeded, rewrote `provider = .yahoo`, and Tiingo was never asked
// again. 22 of 32 cached symbols had drifted off Tiingo that way.
test "applyTiingoCoverage: .covered clears an armed Tiingo backoff" {
const armed = cache.Store.CandleMeta{
.last_close = 100,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .yahoo,
.fail_count = 2,
.tiingo_retry_after_s = 1_790_000_000,
};
const next = DataService.applyTiingoCoverage(armed, "SMPL", 1_787_000_000, .tiingo, .covered);
// Tiingo just served the symbol, so the prior 404 is stale news.
try std.testing.expectEqual(@as(i64, 0), next.tiingo_retry_after_s);
try std.testing.expectEqual(cache.Store.CandleProvider.tiingo, next.provider);
// A successful fetch also resets the transient-failure counter.
try std.testing.expectEqual(@as(u8, 0), next.fail_count);
}
test "applyTiingoCoverage: .not_found arms a jittered backoff ~30 days out" {
const now_s: i64 = 1_787_000_000;
const meta = cache.Store.CandleMeta{
.last_close = 100,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .tiingo,
};
const next = DataService.applyTiingoCoverage(meta, "SMPL", now_s, .yahoo, .not_found);
// Yahoo answered, so provenance records Yahoo...
try std.testing.expectEqual(cache.Store.CandleProvider.yahoo, next.provider);
// ...and the backoff lands 30 days out, +/- the jitter window.
const base = now_s + cache.Ttl.tiingo_backoff;
const max_offset = @divFloor(cache.Ttl.tiingo_backoff * @as(i64, tiingo_backoff_jitter_pct), 100);
try std.testing.expect(next.tiingo_retry_after_s >= base - max_offset);
try std.testing.expect(next.tiingo_retry_after_s <= base + max_offset);
// The spread must be meaningful but bounded: roughly +/-2 days.
try std.testing.expect(max_offset >= 2 * std.time.s_per_day);
try std.testing.expect(max_offset <= 3 * std.time.s_per_day);
}
test "applyTiingoCoverage: .unknown leaves the backoff untouched" {
// A 400 / 402 / malformed body says nothing about coverage. It
// must neither arm a backoff (that's the drift bug) nor clear an
// existing one (that would defeat the 404 we already recorded).
const now_s: i64 = 1_787_000_000;
const unarmed = cache.Store.CandleMeta{
.last_close = 100,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .tiingo,
};
const a = DataService.applyTiingoCoverage(unarmed, "SMPL", now_s, .yahoo, .unknown);
try std.testing.expectEqual(@as(i64, 0), a.tiingo_retry_after_s);
var armed = unarmed;
armed.tiingo_retry_after_s = 1_790_000_000;
const b = DataService.applyTiingoCoverage(armed, "SMPL", now_s, .yahoo, .unknown);
try std.testing.expectEqual(@as(i64, 1_790_000_000), b.tiingo_retry_after_s);
}
test "applyTiingoCoverage: backoff jitter is deterministic per symbol" {
// Deterministic-by-symbol (Wyhash via computeExpires), NOT random:
// repeated writes for the same symbol must not drift the deadline,
// and distinct symbols must land on different days so a batch
// demoted in one window does not all re-probe together.
const now_s: i64 = 1_787_000_000;
const meta = cache.Store.CandleMeta{
.last_close = 100,
.last_date = Date.fromYmd(2026, 8, 14),
.provider = .tiingo,
};
const a1 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found);
const a2 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found);
try std.testing.expectEqual(a1.tiingo_retry_after_s, a2.tiingo_retry_after_s);
// Spread check across a handful of symbols: they must not all
// collapse onto the same instant.
const syms = [_][]const u8{ "SMPLA", "SMPLB", "SMPLC", "SMPLD", "SMPLE", "SMPLF" };
var seen: [syms.len]i64 = undefined;
for (syms, 0..) |sym, i| {
seen[i] = DataService.applyTiingoCoverage(meta, sym, now_s, .yahoo, .not_found).tiingo_retry_after_s;
}
var distinct: usize = 0;
for (seen, 0..) |v, i| {
var is_new = true;
for (seen[0..i]) |prev| {
if (prev == v) is_new = false;
}
if (is_new) distinct += 1;
}
try std.testing.expect(distinct > 1);
}
test "isPermanentProviderFailure gates which Tiingo errors are remembered" {
// The rule that Commit 1 reuses for the Yahoo-demotion decision:
// only a genuine 404 is a statement about the symbol. Everything
// else describes one HTTP call.
try std.testing.expect(isPermanentProviderFailure(error.NotFound));
try std.testing.expect(!isPermanentProviderFailure(error.InvalidResponse));
try std.testing.expect(!isPermanentProviderFailure(error.PaymentRequired));
try std.testing.expect(!isPermanentProviderFailure(error.ParseError));
try std.testing.expect(!isPermanentProviderFailure(error.RateLimited));
try std.testing.expect(!isPermanentProviderFailure(error.Unauthorized));
}
test "loadAllPrices force_refresh tops up without wiping the candle cache" {
// Regression: force_refresh must mean "ignore TTL + incremental
// top-up", NOT "delete the cache and re-download from scratch".
@ -3998,7 +4454,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" {
var candles = [_]Candle{
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
};
store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
store.cacheCandles("HELD", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Any provider/network attempt now panics. If force_refresh wiped
// the cache (old behavior), getCandles would fall through to a full
@ -5114,7 +5570,7 @@ test "serverBarRegression: blocks a regression and reports both dates" {
var s = cache.Store.init(io, allocator, dir_path);
// Local copy holds Monday's bar.
s.updateCandleMeta("AAPL", 100.0, Date.fromYmd(2026, 8, 10), .tiingo, 0, 9999999999);
s.updateCandleMeta("AAPL", .{ .last_close = 100.0, .last_date = Date.fromYmd(2026, 8, 10), .provider = .tiingo }, 9999999999);
const older = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-07,provider::tiingo\n";
const same = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-10,provider::tiingo\n";

View file

@ -1031,12 +1031,36 @@ fn loadData(state: *State, app: *App) void {
};
if (state.view) |*v| v.deinit(app.allocator);
// Splits for the totals row's total-return index. Read straight
// from the cache rather than through a worker: the dividends
// worker above has already warmed the same Tiingo responses that
// carry splits, so this is a local file read. The slices only need
// to outlive `buildReview`, which reduces them to scalars.
var split_map = std.StringHashMap([]const zfin.Split).init(app.allocator);
defer {
var it = split_map.iterator();
while (it.next()) |entry| {
app.allocator.free(@constCast(entry.value_ptr.*));
}
split_map.deinit();
}
for (summary_ptr.allocations) |a| {
if (app.svc.getCachedSplits(app.allocator, a.symbol)) |spl| {
split_map.put(a.symbol, spl.data) catch {
app.allocator.free(spl.data);
break;
};
}
}
state.view = review_view.buildReview(
app.allocator,
app.io,
summary_ptr.*,
candle_map,
dividend_map,
&split_map,
pf,
cm_ptr.*,
acct_map_opt,
@ -2426,6 +2450,7 @@ test "deinitState: cleans up view (leak check)" {
summary,
&candle_map,
null,
null,
portfolio,
cm,
null,

View file

@ -12,6 +12,7 @@ const forecast = @import("../analytics/forecast_evaluation.zig");
const timeline = @import("../analytics/timeline.zig");
const valuation = @import("../analytics/valuation.zig");
const zfin = @import("../root.zig");
const sliceCandlesAsOf = @import("../models/candle.zig").sliceCandlesAsOf;
const snapshot_model = @import("../models/snapshot.zig");
const history = @import("../history.zig");
const Date = @import("../Date.zig");
@ -823,10 +824,9 @@ fn buildContextFromParts(
cd_value,
);
// Fetch benchmark candles (checks cache first). In historical
// mode we slice to `<= as_of` - `performance.trailingReturns`
// anchors on the last candle's date, so trimming the tail gives
// returns "as of" that date for free.
// Fetch benchmark candles (checks cache first). `totalReturns`
// truncates to `<= as_of` internally, so historical mode needs no
// pre-slicing here.
//
// Symbols default to SPY/AGG; user can override via
// `type::config,benchmark_stock::SYMBOL` and
@ -835,36 +835,50 @@ fn buildContextFromParts(
const bond_sym = config.benchmark_bond;
const spy_result = svc.getCandles(stock_sym, .{}) catch null;
defer if (spy_result) |r| r.deinit();
const spy_candles = history.sliceCandlesAsOf(
if (spy_result) |r| r.data else &.{},
as_of,
);
const spy_divs = svc.getCachedDividends(alloc, stock_sym);
defer if (spy_divs) |d| d.deinit();
const agg_result = svc.getCandles(bond_sym, .{}) catch null;
defer if (agg_result) |r| r.deinit();
const agg_candles = history.sliceCandlesAsOf(
const agg_divs = svc.getCachedDividends(alloc, bond_sym);
defer if (agg_divs) |d| d.deinit();
// Total return, not the bare adj_close ratio. This path used to
// call `performance.trailingReturns` directly, which is the
// adj_close series alone - so a stale adjustment basis understated
// the benchmark, the portfolio-weighted return, and
// `conservative_return`, which drives the Monte Carlo projections.
const spy_trailing = performance.totalReturns(
if (spy_result) |r| r.data else &.{},
if (spy_divs) |d| d.data else null,
as_of,
);
const agg_trailing = performance.totalReturns(
if (agg_result) |r| r.data else &.{},
if (agg_divs) |d| d.data else null,
as_of,
);
const spy_trailing = performance.trailingReturns(spy_candles);
const agg_trailing = performance.trailingReturns(agg_candles);
// Build per-position trailing returns from cached candles, each
// optionally truncated to the as-of date. `trailingReturns`
// populates `.week` per position; `portfolioWeightedReturns`
// aggregates the week the same way as the longer periods.
// Build per-position trailing returns from cached candles.
// `totalReturns` populates `.week` per position;
// `portfolioWeightedReturns` aggregates the week the same way as
// the longer periods.
var pos_returns: std.ArrayListUnmanaged(benchmark.PositionReturn) = .empty;
defer pos_returns.deinit(alloc);
for (allocations) |a| {
const candles_res = svc.getCachedCandles(alloc, a.symbol) orelse continue;
defer candles_res.deinit();
const candles = history.sliceCandlesAsOf(candles_res.data, as_of);
if (candles.len > 0) {
const pos_divs = svc.getCachedDividends(alloc, a.symbol);
defer if (pos_divs) |d| d.deinit();
if (sliceCandlesAsOf(candles_res.data, as_of).len > 0) {
try pos_returns.append(alloc, .{
.symbol = a.symbol,
.weight = a.weight,
.returns = performance.trailingReturns(candles),
.returns = performance.totalReturns(
candles_res.data,
if (pos_divs) |d| d.data else null,
as_of,
),
});
}
}

View file

@ -186,8 +186,10 @@ pub const ReviewView = struct {
/// - `dividend_map` (optional): symbol -> dividend slices, used to
/// compute total-return numbers. Pass `null` to fall back to
/// adj_close-derived returns. (Most providers bake dividends into
/// adj_close, so the fallback is usually fine - `withDividendFallback`
/// picks whichever number is higher per period.)
/// adj_close, so the fallback is usually fine, but it inherits any
/// staleness in the cached adjustment basis - see
/// `CandleMeta.adj_basis`. `performance.totalReturnsMonthEnd` picks
/// whichever number is higher per period.)
/// - `classifications`: parsed `metadata.srf`
/// - `account_map` (optional): parsed `accounts.srf`. When null,
/// Tax% is reported as null on every row.
@ -199,6 +201,11 @@ pub fn buildReview(
summary: valuation.PortfolioSummary,
candle_map: *const std.StringHashMap([]const zfin.Candle),
dividend_map: ?*const std.StringHashMap([]const zfin.Dividend),
/// Per-symbol splits, newest-first. Travels with `dividend_map`:
/// the totals row builds a total-return index from raw `close`,
/// which is not split-adjusted, so dividends without splits would
/// read a 2:1 split as a -50% month.
split_map: ?*const std.StringHashMap([]const zfin.Split),
portfolio: zfin.Portfolio,
classifications: classification.ClassificationMap,
account_map: ?analysis.AccountMap,
@ -233,7 +240,7 @@ pub fn buildReview(
const bucket = bucketForSymbol(a.symbol, classifications);
const tax_pct = computeTaxPct(a.symbol, portfolio, account_map, as_of);
const tr_returns = computeTrailingReturns(candles, dividends, as_of);
const tr_returns = performance.totalReturnsMonthEnd(candles, dividends, as_of);
const tr_risk = if (candles.len > 0) risk.trailingRisk(candles) else risk.TrailingRisk{};
try rows.append(allocator, .{
@ -257,6 +264,11 @@ pub fn buildReview(
.symbol = a.symbol,
.candles = candles,
.weight = a.weight,
// Supplying both switches the synthetic series from
// `adj_close` to a total-return index; supplying neither
// leaves it on `adj_close`. Never supply just one.
.dividends = dividends orelse &.{},
.splits = if (split_map) |sm| (sm.get(a.symbol) orelse &.{}) else &.{},
});
}
@ -583,24 +595,6 @@ fn accountIsKnown(am: analysis.AccountMap, account: []const u8) bool {
return false;
}
/// Compute month-end total-return trailing returns. Falls back to
/// adj_close-only when no dividend data is available, picking the
/// higher value per period via `withDividendFallback`.
fn computeTrailingReturns(
candles: []const zfin.Candle,
dividends: ?[]const zfin.Dividend,
as_of: Date,
) performance.TrailingReturns {
if (candles.len == 0) return .{};
const adj = performance.trailingReturnsMonthEnd(candles, as_of);
if (dividends) |divs| {
const div = performance.trailingReturnsMonthEndWithDividends(candles, divs, as_of);
return performance.withDividendFallback(div, adj);
}
return adj;
}
/// Pull the annualized return out of a `PerformanceResult`. For 1Y,
/// the period is right at the threshold - `annualizedReturn` returns
/// null when the actual span < 0.95 years. We accept that null and
@ -844,58 +838,6 @@ test "sortRows: every numeric SortField variant is reachable via extractFloat" {
}
}
test "computeTrailingReturns: empty candles returns empty struct" {
const tr = computeTrailingReturns(&.{}, null, Date.fromYmd(2026, 6, 1));
try testing.expect(tr.one_year == null);
try testing.expect(tr.three_year == null);
try testing.expect(tr.five_year == null);
try testing.expect(tr.ten_year == null);
}
test "computeTrailingReturns: candles with no dividends uses adj_close path" {
// 36 months of 1% monthly growth -> ~12.7% annualized. Without
// dividends, this exercises the adj-close-only branch. Even when
// the synthetic data isn't enough to land any specific trailing
// window (depends on calendar arithmetic against the as_of), the
// function shouldn't crash and should return a struct.
var candles: [36]zfin.Candle = undefined;
var d = Date.fromYmd(2023, 1, 31);
for (0..36) |i| {
const price: f64 = 100.0 * std.math.pow(f64, 1.01, @as(f64, @floatFromInt(i)));
candles[i] = .{ .date = d, .open = price, .high = price, .low = price, .close = price, .adj_close = price, .volume = 1000 };
d = d.addDays(30);
}
const tr = computeTrailingReturns(&candles, null, Date.fromYmd(2026, 1, 31));
// We don't assert on which windows populated - that depends on
// calendar arithmetic against `as_of` and the candle density.
// Coverage of the no-dividends branch (the `else` arm at the
// end of `computeTrailingReturns`) is what we want.
_ = tr;
}
test "computeTrailingReturns: candles with dividends prefers higher fallback" {
// Same shape, but include a dividends slice. The fallback logic
// takes whichever produces a higher annualized return per period;
// for adj_close already including dividends and an explicit
// dividends slice, the helper picks whichever is higher. Either
// way the path through `withDividendFallback` is exercised.
var candles: [36]zfin.Candle = undefined;
var d = Date.fromYmd(2023, 1, 31);
for (0..36) |i| {
const price: f64 = 100.0 + @as(f64, @floatFromInt(i));
candles[i] = .{ .date = d, .open = price, .high = price, .low = price, .close = price, .adj_close = price, .volume = 1000 };
d = d.addDays(30);
}
const divs = [_]zfin.Dividend{
.{ .ex_date = Date.fromYmd(2023, 6, 15), .pay_date = Date.fromYmd(2023, 7, 1), .amount = 0.5 },
.{ .ex_date = Date.fromYmd(2024, 6, 15), .pay_date = Date.fromYmd(2024, 7, 1), .amount = 0.5 },
};
const tr = computeTrailingReturns(&candles, &divs, Date.fromYmd(2026, 1, 31));
// We don't need to assert exact values - coverage of the
// dividend-fallback branch is the goal.
_ = tr;
}
test "computeTaxPct: returns null when account_map is null" {
var lots = [_]zfin.Lot{
.{ .symbol = "VTI", .shares = 100, .open_date = Date.fromYmd(2022, 1, 10), .open_price = 200, .account = "Brokerage" },
@ -1274,6 +1216,7 @@ test "buildReview: dividends_missing separates a failed fetch from a holding tha
summary,
&candle_map,
&dividend_map,
null,
portfolio,
cm,
null,
@ -1303,6 +1246,7 @@ test "buildReview: dividends_missing separates a failed fetch from a holding tha
summary,
&candle_map,
null,
null,
portfolio,
cm,
null,
@ -1402,6 +1346,7 @@ test "buildReview: end-to-end with testing allocator (leak check)" {
summary,
&candle_map,
null, // no dividend map
null, // no split map
portfolio,
cm,
am,