Compare commits
6 commits
44e8c65df4
...
e3ce8f32a1
| Author | SHA1 | Date | |
|---|---|---|---|
| e3ce8f32a1 | |||
| 337d346421 | |||
| 6674969001 | |||
| 7163610b8e | |||
| c4680dbe45 | |||
| 6c75ebf1ec |
15 changed files with 2033 additions and 354 deletions
|
|
@ -56,7 +56,7 @@ pub const BenchmarkComparison = struct {
|
|||
|
||||
/// Per-position data needed for weighted return calculations.
|
||||
/// `returns.week` carries the 1-week return, populated automatically
|
||||
/// by `performance.trailingReturns`.
|
||||
/// by `performance.totalReturns`.
|
||||
pub const PositionReturn = struct {
|
||||
symbol: []const u8,
|
||||
weight: f64,
|
||||
|
|
@ -338,7 +338,7 @@ pub fn buildComparison(
|
|||
) BenchmarkComparison {
|
||||
// `stock_trailing.week` and `bond_trailing.week` propagate
|
||||
// through `toReturnsByPeriod` automatically - see
|
||||
// `performance.trailingReturns`, which populates the field
|
||||
// `performance.totalReturns`, which populates the field
|
||||
// alongside the longer trailing periods.
|
||||
const stock_r = toReturnsByPeriod(stock_trailing);
|
||||
const bond_r = toReturnsByPeriod(bond_trailing);
|
||||
|
|
@ -621,7 +621,7 @@ test "conservativeWeightedReturn single position single period" {
|
|||
|
||||
test "buildComparison with week returns" {
|
||||
// Week returns now flow through `TrailingReturns.week` rather
|
||||
// than separate parameters - `performance.trailingReturns`
|
||||
// than separate parameters - `performance.totalReturns`
|
||||
// populates the field automatically.
|
||||
const stock_tr = TrailingReturns{
|
||||
.one_year = makePR(0.20, 0.20),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
const std = @import("std");
|
||||
const Date = @import("../Date.zig");
|
||||
const Candle = @import("../models/candle.zig").Candle;
|
||||
const sliceCandlesAsOf = @import("../models/candle.zig").sliceCandlesAsOf;
|
||||
const Dividend = @import("../models/dividend.zig").Dividend;
|
||||
const Split = @import("../models/split.zig").Split;
|
||||
const cumulativeSplitRatio = @import("../models/split.zig").cumulativeSplitRatio;
|
||||
|
|
@ -161,7 +162,12 @@ pub const TrailingReturns = struct {
|
|||
/// - When dividend data is complete: reinvestment >= adj_close (compounding)
|
||||
/// - When dividend data is incomplete: adj_close > reinvestment (missing dividends)
|
||||
/// So the higher value is always the more correct one.
|
||||
pub fn withDividendFallback(div_returns: TrailingReturns, adj_close_returns: TrailingReturns) TrailingReturns {
|
||||
///
|
||||
/// Private: reachable only through `totalReturns` /
|
||||
/// `totalReturnsMonthEnd`, which is the point - the merge is not
|
||||
/// optional, and a caller able to skip it can produce a total return
|
||||
/// that is quietly low whenever the cached adjustment basis is stale.
|
||||
fn withDividendFallback(div_returns: TrailingReturns, adj_close_returns: TrailingReturns) TrailingReturns {
|
||||
return .{
|
||||
.one_year = bestResult(div_returns.one_year, adj_close_returns.one_year),
|
||||
.three_year = bestResult(div_returns.three_year, adj_close_returns.three_year),
|
||||
|
|
@ -183,9 +189,77 @@ fn bestResult(a: ?PerformanceResult, b: ?PerformanceResult) ?PerformanceResult {
|
|||
return if (a_ann >= b_ann) a else b;
|
||||
}
|
||||
|
||||
// ── Total return (the public entry points) ───────────────────
|
||||
//
|
||||
// There are two ways to compute a total return from cached data and
|
||||
// neither is reliable alone:
|
||||
//
|
||||
// - The `adj_close` ratio inherits whatever adjustment basis the
|
||||
// cache holds. When a distribution has gone ex since the last full
|
||||
// fetch, the bars behind it were never marked down, so the ratio
|
||||
// reads low by roughly the missed yield (see `CandleMeta.adj_basis`).
|
||||
// - Simulated reinvestment over raw `close` is exact, but only if the
|
||||
// dividend record is complete. A missing dividend silently drops
|
||||
// from the compounding.
|
||||
//
|
||||
// The failure modes are opposite and both are downward, so taking the
|
||||
// higher of the two is correct in both directions. That merge is the
|
||||
// only sanctioned way to get a total return, which is why these are the
|
||||
// only public entry points and the four single-engine wrappers below
|
||||
// are private: a caller reaching for the adj_close series alone gets a
|
||||
// number that is quietly wrong exactly when the cache is stale.
|
||||
//
|
||||
// Both take `as_of` and truncate internally. Previously the as-of
|
||||
// behaviour depended on whether the caller remembered to pre-slice its
|
||||
// candles, which two of three call sites did and one did not.
|
||||
|
||||
/// Total return over 1/3/5/10-year windows ending at the newest bar on
|
||||
/// or before `as_of`.
|
||||
///
|
||||
/// Pass `dividends = null` when no dividend record is available; the
|
||||
/// result then falls back to the adj_close series alone, which is the
|
||||
/// best available answer but is exposed to adjustment-basis staleness.
|
||||
pub fn totalReturns(
|
||||
candles: []const Candle,
|
||||
dividends: ?[]const Dividend,
|
||||
as_of: Date,
|
||||
) TrailingReturns {
|
||||
const window = sliceCandlesAsOf(candles, as_of);
|
||||
if (window.len == 0) return .{};
|
||||
|
||||
const adj = trailingReturns(window);
|
||||
const divs = dividends orelse return adj;
|
||||
return withDividendFallback(trailingReturnsWithDividends(window, divs), adj);
|
||||
}
|
||||
|
||||
/// Morningstar-style total return using month-end reference dates: the
|
||||
/// window ends on the last trading day of the month before `as_of`.
|
||||
///
|
||||
/// Same dividend-vs-adj_close merge as `totalReturns`. Note the start
|
||||
/// dates snap *backward* here (Morningstar's convention) where
|
||||
/// `totalReturns` snaps forward, and `.week` is left null because a
|
||||
/// month-end reference makes a 7-day window meaningless.
|
||||
pub fn totalReturnsMonthEnd(
|
||||
candles: []const Candle,
|
||||
dividends: ?[]const Dividend,
|
||||
as_of: Date,
|
||||
) TrailingReturns {
|
||||
const window = sliceCandlesAsOf(candles, as_of);
|
||||
if (window.len == 0) return .{};
|
||||
|
||||
const adj = trailingReturnsMonthEnd(window, as_of);
|
||||
const divs = dividends orelse return adj;
|
||||
return withDividendFallback(trailingReturnsMonthEndWithDividends(window, divs, as_of), adj);
|
||||
}
|
||||
|
||||
/// Trailing returns from exact calendar date N years ago to latest candle date.
|
||||
/// Start dates snap forward to the next trading day (e.g., weekend -> Monday).
|
||||
pub fn trailingReturns(candles: []const Candle) TrailingReturns {
|
||||
///
|
||||
/// Private: the adj_close-only series understates total return whenever the
|
||||
/// cached adjustment basis predates a distribution, so nothing outside this
|
||||
/// module should consume it directly. Use `totalReturns`, which pairs it with
|
||||
/// the dividend-reinvestment series. See that function's doc comment.
|
||||
fn trailingReturns(candles: []const Candle) TrailingReturns {
|
||||
if (candles.len == 0) return .{};
|
||||
|
||||
const end_date = candles[candles.len - 1].date;
|
||||
|
|
@ -200,7 +274,10 @@ pub fn trailingReturns(candles: []const Candle) TrailingReturns {
|
|||
}
|
||||
|
||||
/// Same as trailingReturns but with dividend reinvestment.
|
||||
pub fn trailingReturnsWithDividends(
|
||||
///
|
||||
/// Private: use `totalReturns`, which merges this with the adj_close
|
||||
/// series so an incomplete dividend record cannot understate the result.
|
||||
fn trailingReturnsWithDividends(
|
||||
candles: []const Candle,
|
||||
dividends: []const Dividend,
|
||||
) TrailingReturns {
|
||||
|
|
@ -221,7 +298,9 @@ pub fn trailingReturnsWithDividends(
|
|||
/// End date = last calendar day of prior month. Start date = that month-end minus N years.
|
||||
/// Both dates snap backward to the last trading day on or before, matching
|
||||
/// Morningstar's "last business day of the month" convention.
|
||||
pub fn trailingReturnsMonthEnd(candles: []const Candle, as_of: Date) TrailingReturns {
|
||||
///
|
||||
/// Private: use `totalReturnsMonthEnd`. See `totalReturns`.
|
||||
fn trailingReturnsMonthEnd(candles: []const Candle, as_of: Date) TrailingReturns {
|
||||
if (candles.len == 0) return .{};
|
||||
|
||||
// End reference = last day of the prior month (snaps backward to last trading day)
|
||||
|
|
@ -236,7 +315,9 @@ pub fn trailingReturnsMonthEnd(candles: []const Candle, as_of: Date) TrailingRet
|
|||
}
|
||||
|
||||
/// Same as trailingReturnsMonthEnd but with dividend reinvestment.
|
||||
pub fn trailingReturnsMonthEndWithDividends(
|
||||
///
|
||||
/// Private: use `totalReturnsMonthEnd`. See `totalReturns`.
|
||||
fn trailingReturnsMonthEndWithDividends(
|
||||
candles: []const Candle,
|
||||
dividends: []const Dividend,
|
||||
as_of: Date,
|
||||
|
|
@ -1002,3 +1083,143 @@ test "trailingReturnsPriceOnly -- price-only diverges from total return on divid
|
|||
// above demonstrates.)
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 0.0), total.one_year.?.total_return, 0.01);
|
||||
}
|
||||
|
||||
// ── totalReturns / totalReturnsMonthEnd (the assemblers) ─────
|
||||
//
|
||||
// These replaced three divergent call-site assemblies: `service.zig`
|
||||
// merged adj_close with dividends for both as-of and month-end,
|
||||
// `views/review.zig` did the same for month-end only, and
|
||||
// `views/projections.zig` did no merge at all - it called the
|
||||
// adj_close-only series directly, so a stale adjustment basis flowed
|
||||
// straight into the benchmark comparison and the Monte Carlo inputs.
|
||||
|
||||
/// 500 consecutive daily bars at a flat price, so a price return of
|
||||
/// exactly zero isolates the dividend contribution.
|
||||
fn flatSeries(buf: []Candle, first: Date, price: f64) []Candle {
|
||||
for (buf, 0..) |*c, i| c.* = makeCandle(first.addDays(@intCast(i)), price);
|
||||
return buf;
|
||||
}
|
||||
|
||||
test "totalReturns: empty candles yields an empty struct" {
|
||||
const tr = totalReturns(&.{}, null, Date.fromYmd(2026, 6, 1));
|
||||
try std.testing.expect(tr.one_year == null);
|
||||
try std.testing.expect(tr.three_year == null);
|
||||
try std.testing.expect(tr.five_year == null);
|
||||
try std.testing.expect(tr.ten_year == null);
|
||||
try std.testing.expect(tr.week == null);
|
||||
}
|
||||
|
||||
test "totalReturns: null dividends degrades to the adj_close series" {
|
||||
// The no-dividend-record case must still produce a number rather
|
||||
// than null - it is the best available answer, just exposed to
|
||||
// adjustment-basis staleness.
|
||||
var buf: [500]Candle = undefined;
|
||||
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
|
||||
|
||||
const tr = totalReturns(candles, null, Date.fromYmd(2026, 8, 14));
|
||||
try std.testing.expect(tr.one_year != null);
|
||||
// Flat price, no dividends: zero total return.
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 0), tr.one_year.?.total_return, 1e-9);
|
||||
}
|
||||
|
||||
test "totalReturns: recovers a distribution the adjustment basis missed" {
|
||||
// The regression this whole change exists for. A series whose
|
||||
// adj_close was never restated has adj_close == close throughout, so
|
||||
// the adj_close ratio equals the *price* return and silently omits
|
||||
// the distribution. The dividend-reinvestment series still sees it,
|
||||
// and the merge must prefer that.
|
||||
var buf: [500]Candle = undefined;
|
||||
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
|
||||
const as_of = Date.fromYmd(2026, 8, 14);
|
||||
|
||||
// $1.00 on a $100 close = a 1% yield, inside the 1Y window.
|
||||
const divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 1, 15), .amount = 1.0 }};
|
||||
|
||||
const stale = totalReturns(candles, null, as_of);
|
||||
const merged = totalReturns(candles, &divs, as_of);
|
||||
|
||||
// Uncorrected, the distribution is invisible.
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 0), stale.one_year.?.total_return, 1e-9);
|
||||
// Merged, it is worth exactly its yield.
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 0.01), merged.one_year.?.total_return, 1e-9);
|
||||
try std.testing.expect(merged.one_year.?.total_return > stale.one_year.?.total_return);
|
||||
}
|
||||
|
||||
test "totalReturns: a complete dividend record is never worse than adj_close" {
|
||||
// `withDividendFallback` takes the higher of the two per period, so
|
||||
// adding dividends can only raise a total return, never lower it.
|
||||
// That directionality is what makes the merge safe: both failure
|
||||
// modes (stale basis, incomplete dividends) bias downward.
|
||||
var buf: [500]Candle = undefined;
|
||||
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
|
||||
const as_of = Date.fromYmd(2026, 8, 14);
|
||||
const divs = [_]Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2025, 9, 15), .amount = 0.5 },
|
||||
.{ .ex_date = Date.fromYmd(2026, 3, 15), .amount = 0.5 },
|
||||
};
|
||||
|
||||
const without = totalReturns(candles, null, as_of);
|
||||
const with = totalReturns(candles, &divs, as_of);
|
||||
try std.testing.expect(with.one_year.?.total_return >= without.one_year.?.total_return);
|
||||
}
|
||||
|
||||
test "totalReturns: as_of truncates the window instead of using the newest bar" {
|
||||
// Previously the as-of behaviour depended on whether the caller
|
||||
// remembered to pre-slice - `views/projections.zig` did,
|
||||
// `service.zig` did not. Now it is the function's job.
|
||||
var buf: [500]Candle = undefined;
|
||||
// Rising 1/day from 100, so the endpoint is identifiable by value.
|
||||
for (&buf, 0..) |*c, i| {
|
||||
c.* = makeCandle(Date.fromYmd(2025, 6, 1).addDays(@intCast(i)), 100.0 + @as(f64, @floatFromInt(i)));
|
||||
}
|
||||
const candles = buf[0..];
|
||||
|
||||
const late = totalReturns(candles, null, Date.fromYmd(2026, 8, 14));
|
||||
const early = totalReturns(candles, null, Date.fromYmd(2026, 6, 14));
|
||||
|
||||
// Both resolve a 1Y window...
|
||||
try std.testing.expect(late.one_year != null);
|
||||
try std.testing.expect(early.one_year != null);
|
||||
// ...ending on their own as_of, not on the last bar in the slice.
|
||||
try std.testing.expect(late.one_year.?.to.eql(Date.fromYmd(2026, 8, 14)));
|
||||
try std.testing.expect(early.one_year.?.to.eql(Date.fromYmd(2026, 6, 14)));
|
||||
}
|
||||
|
||||
test "totalReturns: as_of before all candles yields an empty struct" {
|
||||
var buf: [500]Candle = undefined;
|
||||
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
|
||||
const tr = totalReturns(candles, null, Date.fromYmd(2020, 1, 1));
|
||||
try std.testing.expect(tr.one_year == null);
|
||||
try std.testing.expect(tr.week == null);
|
||||
}
|
||||
|
||||
test "totalReturnsMonthEnd: window ends on the last bar of the prior month" {
|
||||
// Morningstar convention, and the reason this variant exists
|
||||
// separately: the endpoint is a month boundary, not `as_of` itself.
|
||||
var buf: [500]Candle = undefined;
|
||||
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
|
||||
|
||||
const tr = totalReturnsMonthEnd(candles, null, Date.fromYmd(2026, 8, 14));
|
||||
try std.testing.expect(tr.one_year != null);
|
||||
try std.testing.expect(tr.one_year.?.to.eql(Date.fromYmd(2026, 7, 31)));
|
||||
// A month-end reference makes a 7-day window meaningless.
|
||||
try std.testing.expect(tr.week == null);
|
||||
}
|
||||
|
||||
test "totalReturnsMonthEnd: recovers a missed distribution too" {
|
||||
var buf: [500]Candle = undefined;
|
||||
const candles = flatSeries(&buf, Date.fromYmd(2025, 6, 1), 100);
|
||||
const as_of = Date.fromYmd(2026, 8, 14);
|
||||
const divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 1, 15), .amount = 1.0 }};
|
||||
|
||||
const stale = totalReturnsMonthEnd(candles, null, as_of);
|
||||
const merged = totalReturnsMonthEnd(candles, &divs, as_of);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 0), stale.one_year.?.total_return, 1e-9);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 0.01), merged.one_year.?.total_return, 1e-9);
|
||||
}
|
||||
|
||||
test "totalReturnsMonthEnd: empty candles yields an empty struct" {
|
||||
const tr = totalReturnsMonthEnd(&.{}, null, Date.fromYmd(2026, 6, 1));
|
||||
try std.testing.expect(tr.one_year == null);
|
||||
try std.testing.expect(tr.ten_year == null);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ const std = @import("std");
|
|||
const Date = @import("../Date.zig");
|
||||
const Candle = @import("../models/candle.zig").Candle;
|
||||
const risk = @import("risk.zig");
|
||||
const Dividend = @import("../models/dividend.zig").Dividend;
|
||||
const Split = @import("../models/split.zig").Split;
|
||||
|
||||
/// Per-window flags marking metrics that required holding-dropout
|
||||
/// renormalization. Set when at least one position lacked candle coverage
|
||||
|
|
@ -87,8 +89,36 @@ pub const PositionCandles = struct {
|
|||
/// Position weight in the portfolio (market_value / total_value).
|
||||
/// Must be non-negative; zero-weight positions are skipped entirely.
|
||||
weight: f64,
|
||||
/// Cash distributions, **newest first** - the order
|
||||
/// `cache.Store.read` returns them in, since `writeMerged` sorts
|
||||
/// descending by date.
|
||||
///
|
||||
/// When this and `splits` are both empty, the resample falls back to
|
||||
/// the provider's `adj_close`, preserving the behaviour callers had
|
||||
/// before these fields existed. Supply them to get a total-return
|
||||
/// index built from raw `close` instead - see `synthesizeWindow`.
|
||||
dividends: []const Dividend = &.{},
|
||||
/// Split events, **newest first**, same convention as `dividends`.
|
||||
///
|
||||
/// Required for correctness whenever `dividends` is supplied: raw
|
||||
/// `close` is not split-adjusted, so an index built without these
|
||||
/// would read a 2:1 split as a -50% month.
|
||||
splits: []const Split = &.{},
|
||||
};
|
||||
|
||||
/// Debug-only guard on the newest-first ordering the resample relies on.
|
||||
/// A mis-ordered slice would silently mis-time reinvestment rather than
|
||||
/// fail, so catch it where tests will see it.
|
||||
fn assertDescending(comptime T: type, items: []const T, comptime dateField: []const u8) void {
|
||||
if (!std.debug.runtime_safety) return;
|
||||
if (items.len < 2) return;
|
||||
for (items[1..], 0..) |item, i| {
|
||||
const newer = @field(items[i], dateField);
|
||||
const older = @field(item, dateField);
|
||||
std.debug.assert(!newer.lessThan(older));
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute true portfolio-level risk metrics for the standard windows.
|
||||
/// Iterates `positions`, derives per-position monthly return series,
|
||||
/// builds a weighted synthetic series per window with dropout-and-
|
||||
|
|
@ -314,20 +344,73 @@ fn synthesizeWindow(
|
|||
|
||||
for (participants_buf[0..n_participants], 0..) |orig_idx, p_idx| {
|
||||
const cand = positions[orig_idx].candles;
|
||||
// Walk candles, recording the LAST close in each month bucket
|
||||
const divs = positions[orig_idx].dividends;
|
||||
const spls = positions[orig_idx].splits;
|
||||
assertDescending(Dividend, divs, "ex_date");
|
||||
assertDescending(Split, spls, "date");
|
||||
|
||||
// Which price series to resample.
|
||||
//
|
||||
// With corporate actions supplied, build a total-return index in
|
||||
// units of shares held: start at one share, multiply on a split,
|
||||
// and buy more with each distribution at that day's close.
|
||||
//
|
||||
// value(t) = shares(t) * close(t)
|
||||
//
|
||||
// Both events pass through that expression continuously - a split
|
||||
// multiplies shares while dividing close, and a distribution buys
|
||||
// exactly the shares its per-share drop paid for - so the series
|
||||
// has no artificial cliffs and month-over-month ratios are true
|
||||
// total returns.
|
||||
//
|
||||
// Why not `adj_close`: it is only as current as the last full
|
||||
// candle fetch (see `CandleMeta.adj_basis`), and Yahoo's parser
|
||||
// leaves it at 0 for a JSON `null`, which the `prev_p <= 0` guard
|
||||
// below then silently drops - losing a whole month from the
|
||||
// series rather than merely mis-stating it.
|
||||
//
|
||||
// Without corporate actions there is nothing to build from, so
|
||||
// fall back to `adj_close` (via `chartClose`, which handles the
|
||||
// zero case). That keeps callers predating these fields on
|
||||
// exactly their old behaviour.
|
||||
const use_index = divs.len > 0 or spls.len > 0;
|
||||
|
||||
// Walk candles, recording the LAST value in each month bucket
|
||||
// that falls within the window. We use yearMonth() for the
|
||||
// month-boundary comparison (cheap integer compare) and
|
||||
// Date.monthsBetween for the slot index into `prices`.
|
||||
var prev_ym: u32 = 0;
|
||||
var prev_date: Date = window_start;
|
||||
var last_close: f64 = 0;
|
||||
var last_value: f64 = 0;
|
||||
var have_any = false;
|
||||
|
||||
// Corporate-action cursors. Both slices are newest-first, so walk
|
||||
// them from the tail to consume events chronologically. Pre-window
|
||||
// bars are walked too, which is what makes `shares` correct on
|
||||
// entry to the window.
|
||||
var shares: f64 = 1.0;
|
||||
var di: usize = divs.len;
|
||||
var si: usize = spls.len;
|
||||
|
||||
for (cand) |c| {
|
||||
if (use_index) {
|
||||
// Splits first: a distribution's per-share amount is
|
||||
// quoted against the share count in force on its ex-date.
|
||||
while (si > 0 and !c.date.lessThan(spls[si - 1].date)) {
|
||||
shares *= spls[si - 1].ratio();
|
||||
si -= 1;
|
||||
}
|
||||
while (di > 0 and !c.date.lessThan(divs[di - 1].ex_date)) {
|
||||
if (c.close > 0) shares += divs[di - 1].amount * shares / c.close;
|
||||
di -= 1;
|
||||
}
|
||||
}
|
||||
const value = if (use_index) shares * c.close else c.chartClose();
|
||||
|
||||
if (c.date.days < window_start.days) {
|
||||
// Before window: still track the running last-close,
|
||||
// Before window: still track the running last value,
|
||||
// but only emit it once we cross into the window.
|
||||
last_close = c.adj_close;
|
||||
last_value = value;
|
||||
prev_ym = c.date.yearMonth();
|
||||
prev_date = c.date;
|
||||
have_any = true;
|
||||
|
|
@ -336,27 +419,27 @@ fn synthesizeWindow(
|
|||
if (c.date.days > window_end.days) break;
|
||||
const ym = c.date.yearMonth();
|
||||
if (have_any and ym != prev_ym) {
|
||||
// Month boundary: stash prev month's last close.
|
||||
// Month boundary: stash prev month's last value.
|
||||
const m_idx_signed = Date.monthsBetween(window_start, prev_date);
|
||||
if (m_idx_signed >= 0) {
|
||||
const m_idx: usize = @intCast(m_idx_signed);
|
||||
if (m_idx < n_months_total) {
|
||||
prices[p_idx * n_months_total + m_idx] = last_close;
|
||||
prices[p_idx * n_months_total + m_idx] = last_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
last_close = c.adj_close;
|
||||
last_value = value;
|
||||
prev_ym = ym;
|
||||
prev_date = c.date;
|
||||
have_any = true;
|
||||
}
|
||||
// Final partial month: stash whatever the latest close was.
|
||||
// Final partial month: stash whatever the latest value was.
|
||||
if (have_any) {
|
||||
const m_idx_signed = Date.monthsBetween(window_start, prev_date);
|
||||
if (m_idx_signed >= 0) {
|
||||
const m_idx: usize = @intCast(m_idx_signed);
|
||||
if (m_idx < n_months_total) {
|
||||
prices[p_idx * n_months_total + m_idx] = last_close;
|
||||
prices[p_idx * n_months_total + m_idx] = last_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -702,3 +785,173 @@ test "syntheticPortfolioRisk: return_3y annualizes correctly" {
|
|||
try testing.expect(r.return_3y.? > 0.10);
|
||||
try testing.expect(r.return_3y.? < 0.16);
|
||||
}
|
||||
|
||||
// ── Total-return index (dividends + splits) ──────────────────
|
||||
//
|
||||
// The resample used to read `adj_close` directly. That inherits any
|
||||
// staleness in the cached adjustment basis, and Yahoo's parser leaves
|
||||
// `adj_close` at 0 for a JSON `null`, which the `prev_p <= 0` guard
|
||||
// drops - losing the whole month. Supplying dividends and splits
|
||||
// switches to an index built from raw `close`, in units of shares held.
|
||||
|
||||
/// Build a candle slice with an explicit close per month, so tests can
|
||||
/// place a corporate action against a known price.
|
||||
fn monthlyCloses(allocator: std.mem.Allocator, start_year: i16, closes: []const f64) ![]Candle {
|
||||
var out = std.ArrayList(Candle).empty;
|
||||
errdefer out.deinit(allocator);
|
||||
for (closes, 0..) |px, i| {
|
||||
const year_offset: i16 = @intCast(i / 12);
|
||||
const m_in_year: u8 = @intCast((i % 12) + 1);
|
||||
try out.append(allocator, makeCandle(Date.fromYmd(start_year + year_offset, m_in_year, 15), px));
|
||||
try out.append(allocator, makeCandle(Date.fromYmd(start_year + year_offset, m_in_year, 28), px));
|
||||
}
|
||||
return out.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
test "synthesizeWindow index: a split is not a -50% month" {
|
||||
// The correctness trap in building an index from raw `close`.
|
||||
// `close` is not split-adjusted, so a 2:1 split halves it; without
|
||||
// multiplying the share count the series reads a catastrophic month
|
||||
// and inflates vol. 40 flat months with a split in the middle must
|
||||
// produce zero volatility.
|
||||
const a = testing.allocator;
|
||||
var closes: [40]f64 = undefined;
|
||||
for (&closes, 0..) |*c, i| c.* = if (i < 20) 200.0 else 100.0;
|
||||
const cand = try monthlyCloses(a, 2022, &closes);
|
||||
defer a.free(cand);
|
||||
|
||||
// Split lands on the first bar of the halved run.
|
||||
const splits = [_]Split{.{ .date = Date.fromYmd(2023, 9, 15), .numerator = 2, .denominator = 1 }};
|
||||
const positions = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .splits = &splits },
|
||||
};
|
||||
|
||||
const r = try syntheticPortfolioRisk(a, &positions, Date.fromYmd(2025, 5, 1));
|
||||
try testing.expect(r.vol_3y != null);
|
||||
// A flat total-return series has no volatility. Without the share
|
||||
// multiplication this would be enormous.
|
||||
try testing.expect(r.vol_3y.? < 0.01);
|
||||
try testing.expect(r.return_3y != null);
|
||||
try testing.expectApproxEqAbs(@as(f64, 0), r.return_3y.?, 0.01);
|
||||
}
|
||||
|
||||
test "synthesizeWindow index: distributions raise the total return" {
|
||||
// Flat price, so every bit of return comes from the distributions.
|
||||
const a = testing.allocator;
|
||||
var closes: [40]f64 = undefined;
|
||||
for (&closes) |*c| c.* = 100.0;
|
||||
const cand = try monthlyCloses(a, 2022, &closes);
|
||||
defer a.free(cand);
|
||||
|
||||
// Newest-first, matching the cache's on-disk order.
|
||||
const divs = [_]Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2024, 9, 15), .amount = 1.0 },
|
||||
.{ .ex_date = Date.fromYmd(2024, 3, 15), .amount = 1.0 },
|
||||
.{ .ex_date = Date.fromYmd(2023, 9, 15), .amount = 1.0 },
|
||||
};
|
||||
|
||||
const bare = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0 },
|
||||
};
|
||||
const paying = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &divs },
|
||||
};
|
||||
|
||||
const without = try syntheticPortfolioRisk(a, &bare, Date.fromYmd(2025, 5, 1));
|
||||
const with = try syntheticPortfolioRisk(a, &paying, Date.fromYmd(2025, 5, 1));
|
||||
|
||||
// Flat price and no distributions: zero return.
|
||||
try testing.expectApproxEqAbs(@as(f64, 0), without.return_3y.?, 1e-6);
|
||||
// Three 1% reinvestments inside the window compound to ~3.03%,
|
||||
// annualized over 3 years to ~1%.
|
||||
try testing.expect(with.return_3y.? > 0.005);
|
||||
try testing.expect(with.return_3y.? < 0.015);
|
||||
}
|
||||
|
||||
test "synthesizeWindow index: unusable adj_close no longer drops the month" {
|
||||
// Yahoo yields adj_close == 0 for a JSON null. The `prev_p <= 0`
|
||||
// guard then skips both month transitions touching it, silently
|
||||
// shortening the series. With an index built from `close`, the
|
||||
// zeroed field is never consulted.
|
||||
const a = testing.allocator;
|
||||
var closes: [40]f64 = undefined;
|
||||
for (&closes, 0..) |*c, i| c.* = 100.0 + @as(f64, @floatFromInt(i));
|
||||
const cand = try monthlyCloses(a, 2022, &closes);
|
||||
defer a.free(cand);
|
||||
|
||||
// Zero out adj_close on a mid-window month, as a null would.
|
||||
for (cand) |*c| {
|
||||
if (c.date.yearMonth() == Date.fromYmd(2024, 1, 15).yearMonth()) c.adj_close = 0;
|
||||
}
|
||||
|
||||
const divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2023, 6, 15), .amount = 0.5 }};
|
||||
const bare = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0 },
|
||||
};
|
||||
const indexed = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &divs },
|
||||
};
|
||||
|
||||
const as_of = Date.fromYmd(2025, 5, 1);
|
||||
const without = try syntheticPortfolioRisk(a, &bare, as_of);
|
||||
const with = try syntheticPortfolioRisk(a, &indexed, as_of);
|
||||
|
||||
// Both still produce numbers - `chartClose`'s zero fallback keeps
|
||||
// the bare path alive rather than dropping the month outright.
|
||||
try testing.expect(without.return_3y != null);
|
||||
try testing.expect(with.return_3y != null);
|
||||
// And the indexed path is strictly better: it also counts the
|
||||
// distribution the bare path cannot see.
|
||||
try testing.expect(with.return_3y.? > without.return_3y.?);
|
||||
}
|
||||
|
||||
test "synthesizeWindow index: no corporate actions preserves adj_close behaviour" {
|
||||
// The defaulted fields must be a no-op, or every caller predating
|
||||
// them silently changes numbers.
|
||||
const a = testing.allocator;
|
||||
const cand = try buildMonthlyCandles(a, 2022, 45, &linearGrowth);
|
||||
defer a.free(cand);
|
||||
|
||||
const positions = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0 },
|
||||
};
|
||||
const as_of = Date.fromYmd(2025, 11, 1);
|
||||
const baseline = try syntheticPortfolioRisk(a, &positions, as_of);
|
||||
|
||||
// Same input, explicitly-empty corporate actions.
|
||||
const explicit = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &.{}, .splits = &.{} },
|
||||
};
|
||||
const same = try syntheticPortfolioRisk(a, &explicit, as_of);
|
||||
|
||||
try testing.expectEqual(baseline.return_3y, same.return_3y);
|
||||
try testing.expectEqual(baseline.vol_3y, same.vol_3y);
|
||||
try testing.expectEqual(baseline.sharpe_3y, same.sharpe_3y);
|
||||
}
|
||||
|
||||
test "synthesizeWindow index: split and dividend on the same date" {
|
||||
// Order matters: a distribution's per-share amount is quoted
|
||||
// against the share count in force on its ex-date, so the split
|
||||
// must apply first. Getting it backwards misprices the
|
||||
// reinvestment by the split ratio.
|
||||
const a = testing.allocator;
|
||||
var closes: [40]f64 = undefined;
|
||||
for (&closes, 0..) |*c, i| c.* = if (i < 20) 200.0 else 100.0;
|
||||
const cand = try monthlyCloses(a, 2022, &closes);
|
||||
defer a.free(cand);
|
||||
|
||||
const shared = Date.fromYmd(2023, 9, 15);
|
||||
const splits = [_]Split{.{ .date = shared, .numerator = 2, .denominator = 1 }};
|
||||
const divs = [_]Dividend{.{ .ex_date = shared, .amount = 1.0 }};
|
||||
const positions = [_]PositionCandles{
|
||||
.{ .symbol = "SMPL", .candles = cand, .weight = 1.0, .dividends = &divs, .splits = &splits },
|
||||
};
|
||||
|
||||
const r = try syntheticPortfolioRisk(a, &positions, Date.fromYmd(2025, 5, 1));
|
||||
// Flat total-return apart from a single 1% reinvestment: small
|
||||
// positive return, and still essentially no volatility.
|
||||
try testing.expect(r.return_3y != null);
|
||||
try testing.expect(r.return_3y.? > 0);
|
||||
try testing.expect(r.return_3y.? < 0.01);
|
||||
try testing.expect(r.vol_3y.? < 0.02);
|
||||
}
|
||||
|
|
|
|||
154
src/cache/freshness.zig
vendored
154
src/cache/freshness.zig
vendored
|
|
@ -30,11 +30,30 @@
|
|||
//!
|
||||
//! Pure: dates and `now_s` in, findings out. No I/O, no cache reads, no
|
||||
//! fetches. The caller supplies the corpus.
|
||||
//!
|
||||
//! ## Adjustment-basis staleness
|
||||
//!
|
||||
//! A second, unrelated staleness question lives here too:
|
||||
//! `adjustmentBasisStale` asks whether a symbol's cached `adj_close`
|
||||
//! values predate a corporate action they should already reflect. It is
|
||||
//! the same shape of problem - "this cache entry is quietly frozen" -
|
||||
//! and it follows the same two-part split as the peer analysis above:
|
||||
//! `newestCorporateAction` gathers from disk (like `collect`), and
|
||||
//! `adjustmentBasisStale` decides (like `scan`, pure dates in, bool
|
||||
//! out).
|
||||
//!
|
||||
//! It deliberately does not live in `store.zig`. The cache layer
|
||||
//! serializes and reads; deciding that a distribution behind the basis
|
||||
//! means the series must be refetched is domain policy, and
|
||||
//! `store.zig`'s own `updateCandleMeta` doc records the same boundary
|
||||
//! for market-clock knowledge ("owned by the caller").
|
||||
|
||||
const std = @import("std");
|
||||
const Date = @import("../Date.zig");
|
||||
const market = @import("../market.zig");
|
||||
const cache = @import("store.zig");
|
||||
const Dividend = @import("../models/dividend.zig").Dividend;
|
||||
const Split = @import("../models/split.zig").Split;
|
||||
|
||||
/// One cached symbol, as the caller found it.
|
||||
pub const Entry = struct {
|
||||
|
|
@ -603,3 +622,138 @@ test "collect-style exclusion: an excluded symbol produces no finding at all" {
|
|||
try testing.expectEqual(@as(usize, 0), without.orphans.len);
|
||||
try testing.expectEqual(@as(usize, 0), without.stale.len);
|
||||
}
|
||||
|
||||
// ── Adjustment-basis staleness ───────────────────────────────
|
||||
//
|
||||
// The candle cache is append-only. `getCandles` tops up from
|
||||
// `last_date + 1`, and the newly-appended bars arrive with
|
||||
// `adj_close == close` because nothing has gone ex after them yet -
|
||||
// while every previously-cached bar keeps the adjustment basis it was
|
||||
// originally fetched with. When the next distribution goes ex, the bars
|
||||
// behind it should be marked down by its factor and nothing does it, so
|
||||
// every total return spanning that ex-date reads low by roughly the
|
||||
// missed yield.
|
||||
//
|
||||
// `CandleMeta.adj_basis` records how current a series' basis is. These
|
||||
// two functions turn that into a verdict, split the same way as
|
||||
// `collect` / `scan`: one gathers from disk, one decides.
|
||||
|
||||
/// Newest corporate-action ex-date cached for a symbol, or null when no
|
||||
/// dividends or splits are on disk.
|
||||
///
|
||||
/// Splits count as much as dividends here, and arguably more: raw
|
||||
/// `close` is not split-adjusted (hence `split.cumulativeSplitRatio`),
|
||||
/// so an unapplied 2:1 split leaves older `adj_close` values wrong by
|
||||
/// 50% rather than by a quarter's yield.
|
||||
///
|
||||
/// Allocates only transiently - the returned `Date` borrows nothing.
|
||||
pub fn newestCorporateAction(allocator: std.mem.Allocator, store: *cache.Store, symbol: []const u8) ?Date {
|
||||
var newest: ?Date = null;
|
||||
|
||||
// `CacheResult` has no deinit - the caller owns `data`. Dividends
|
||||
// carry owned strings, so they need `freeSlice`; splits are
|
||||
// pure-numeric.
|
||||
if (store.read(allocator, Dividend, symbol, null, .any)) |r| {
|
||||
defer Dividend.freeSlice(allocator, r.data);
|
||||
for (r.data) |d| {
|
||||
if (newest == null or newest.?.lessThan(d.ex_date)) newest = d.ex_date;
|
||||
}
|
||||
}
|
||||
if (store.read(allocator, Split, symbol, null, .any)) |r| {
|
||||
defer allocator.free(r.data);
|
||||
for (r.data) |sp| {
|
||||
if (newest == null or newest.?.lessThan(sp.date)) newest = sp.date;
|
||||
}
|
||||
}
|
||||
|
||||
return newest;
|
||||
}
|
||||
|
||||
/// Does a series' `adj_close` predate a corporate action it should
|
||||
/// already reflect?
|
||||
///
|
||||
/// - `adj_basis`: `CandleMeta.adj_basis` - the bar date through which
|
||||
/// the cached `adj_close` values reflect corporate actions.
|
||||
/// - `last_bar`: `CandleMeta.last_date` - the newest bar held.
|
||||
/// - `newest_action`: from `newestCorporateAction`, or null for a
|
||||
/// symbol with no cached dividends or splits.
|
||||
///
|
||||
/// The `last_bar` bound is what keeps this convergent. A distribution
|
||||
/// announced with a future ex-date is not yet reflected in *any*
|
||||
/// provider's adjustment series, so treating it as stale would refetch
|
||||
/// on every pass and never settle.
|
||||
///
|
||||
/// Pure: three dates in, bool out.
|
||||
pub fn adjustmentBasisStale(adj_basis: Date, last_bar: Date, newest_action: ?Date) bool {
|
||||
const newest = newest_action orelse return false;
|
||||
if (last_bar.lessThan(newest)) return false;
|
||||
return adj_basis.lessThan(newest);
|
||||
}
|
||||
|
||||
test "adjustmentBasisStale: no corporate action means nothing to restate" {
|
||||
// A non-payer has no ex-date to exceed the basis, so it must never
|
||||
// escalate - not even with the epoch sentinel a legacy cache parses to.
|
||||
try testing.expect(!adjustmentBasisStale(Date.fromYmd(2026, 8, 14), Date.fromYmd(2026, 8, 14), null));
|
||||
try testing.expect(!adjustmentBasisStale(Date.epoch, Date.fromYmd(2026, 8, 14), null));
|
||||
}
|
||||
|
||||
test "adjustmentBasisStale: action behind the basis is stale" {
|
||||
const last_bar = Date.fromYmd(2026, 8, 14);
|
||||
const ex = Date.fromYmd(2026, 6, 18);
|
||||
|
||||
// Basis at the newest bar already covers the ex-date.
|
||||
try testing.expect(!adjustmentBasisStale(last_bar, last_bar, ex));
|
||||
// Basis exactly at the ex-date covers it too (not `lessThan`).
|
||||
try testing.expect(!adjustmentBasisStale(ex, last_bar, ex));
|
||||
// Basis behind the ex-date: the bars in between were never marked down.
|
||||
try testing.expect(adjustmentBasisStale(Date.fromYmd(2026, 5, 1), last_bar, ex));
|
||||
// The legacy-cache case: sentinel basis on a dividend payer. This is
|
||||
// the shape every pre-adj_basis cache lands in.
|
||||
try testing.expect(adjustmentBasisStale(Date.epoch, last_bar, ex));
|
||||
}
|
||||
|
||||
test "adjustmentBasisStale: a not-yet-ex action is not stale" {
|
||||
// Ex-date beyond the newest bar we hold. No provider has applied it
|
||||
// either, so escalating would never converge.
|
||||
try testing.expect(!adjustmentBasisStale(
|
||||
Date.fromYmd(2026, 5, 1),
|
||||
Date.fromYmd(2026, 8, 14),
|
||||
Date.fromYmd(2026, 9, 17),
|
||||
));
|
||||
// Boundary: an ex-date exactly on the newest bar IS covered.
|
||||
try testing.expect(adjustmentBasisStale(
|
||||
Date.fromYmd(2026, 5, 1),
|
||||
Date.fromYmd(2026, 8, 14),
|
||||
Date.fromYmd(2026, 8, 14),
|
||||
));
|
||||
}
|
||||
|
||||
test "newestCorporateAction: takes the max across dividends and splits" {
|
||||
const io = testing.io;
|
||||
const a = testing.allocator;
|
||||
var tmp = testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", a);
|
||||
defer a.free(dir_path);
|
||||
|
||||
var store = cache.Store.init(io, a, dir_path);
|
||||
|
||||
// Nothing cached at all.
|
||||
try testing.expect(newestCorporateAction(a, &store, "SMPL") == null);
|
||||
|
||||
var divs = [_]Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2026, 3, 20), .amount = 1.79 },
|
||||
.{ .ex_date = Date.fromYmd(2026, 6, 18), .amount = 1.90 },
|
||||
};
|
||||
store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
|
||||
try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 6, 18)));
|
||||
|
||||
// A later split must win over the later dividend.
|
||||
var splits = [_]Split{.{ .date = Date.fromYmd(2026, 7, 1), .numerator = 2, .denominator = 1 }};
|
||||
store.write(Split, "SMPL", splits[0..], .{ .seconds = cache.Ttl.splits });
|
||||
try testing.expect(newestCorporateAction(a, &store, "SMPL").?.eql(Date.fromYmd(2026, 7, 1)));
|
||||
|
||||
// Splits alone (different symbol) still resolve.
|
||||
store.write(Split, "SMPLB", splits[0..], .{ .seconds = cache.Ttl.splits });
|
||||
try testing.expect(newestCorporateAction(a, &store, "SMPLB").?.eql(Date.fromYmd(2026, 7, 1)));
|
||||
}
|
||||
|
|
|
|||
410
src/cache/store.zig
vendored
410
src/cache/store.zig
vendored
|
|
@ -67,6 +67,17 @@ pub const Ttl = struct {
|
|||
/// month.
|
||||
pub const tickers_funds: i64 = 30 * s_per_day;
|
||||
pub const tickers_companies: i64 = 30 * s_per_day;
|
||||
|
||||
/// How long a genuine Tiingo 404 suppresses further Tiingo
|
||||
/// attempts for a symbol (see `CandleMeta.tiingo_retry_after_s`).
|
||||
///
|
||||
/// Long enough that a symbol Tiingo genuinely does not carry
|
||||
/// costs about one wasted 404 per month, short enough that
|
||||
/// coverage changes get picked up without operator action.
|
||||
/// Call sites layer `jitter_pct` on top so a batch of symbols
|
||||
/// demoted in the same window does not all re-probe on the same
|
||||
/// day.
|
||||
pub const tiingo_backoff: i64 = 30 * s_per_day;
|
||||
};
|
||||
|
||||
/// Cache TTL specification with optional per-key expiration jitter.
|
||||
|
|
@ -894,7 +905,17 @@ pub const Store = struct {
|
|||
/// candle metadata, stamping its `#!expires=` with `expires_at_s`
|
||||
/// (the caller computes the market-aware freshness boundary; see
|
||||
/// `market.nextCandleExpiry`).
|
||||
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||
///
|
||||
/// `attrs` supplies the provider-state fields; `last_close`,
|
||||
/// `last_date` and `adj_basis` are derived from the newest candle
|
||||
/// written.
|
||||
///
|
||||
/// Deriving `adj_basis` here is the whole point of routing full
|
||||
/// refetches through this function: replacing the file means every
|
||||
/// bar carries the provider's current adjustment basis, which by
|
||||
/// construction reflects every corporate action up to the newest
|
||||
/// bar. `appendCandles` deliberately cannot do this.
|
||||
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, attrs: CandleMetaAttrs, expires_at_s: i64) void {
|
||||
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
|
||||
defer self.allocator.free(srf_data);
|
||||
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
|
||||
|
|
@ -906,7 +927,14 @@ pub const Store = struct {
|
|||
|
||||
if (candles.len > 0) {
|
||||
const last = candles[candles.len - 1];
|
||||
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
|
||||
self.updateCandleMeta(symbol, .{
|
||||
.last_close = last.close,
|
||||
.last_date = last.date,
|
||||
.provider = attrs.provider,
|
||||
.fail_count = attrs.fail_count,
|
||||
.tiingo_retry_after_s = attrs.tiingo_retry_after_s,
|
||||
.adj_basis = last.date,
|
||||
}, expires_at_s);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -914,7 +942,17 @@ pub const Store = struct {
|
|||
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
|
||||
/// Also updates candle metadata, stamping its `#!expires=` with
|
||||
/// `expires_at_s` (caller-computed market-aware boundary).
|
||||
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||
///
|
||||
/// `meta` supplies every metadata field except `last_close` and
|
||||
/// `last_date`, which are derived from the newest appended candle.
|
||||
/// Callers are expected to pass the symbol's *existing* meta so
|
||||
/// that fields describing the series as a whole survive an append
|
||||
/// unchanged - appending bars does not re-derive anything about
|
||||
/// the rows already on disk. `adj_basis` in particular MUST NOT
|
||||
/// advance here: the appended bars do not restate the older rows'
|
||||
/// `adj_close`, so claiming a newer basis would mask exactly the
|
||||
/// staleness that field exists to detect.
|
||||
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, meta: CandleMeta, expires_at_s: i64) void {
|
||||
if (new_candles.len == 0) return;
|
||||
|
||||
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
|
||||
|
|
@ -943,23 +981,25 @@ pub const Store = struct {
|
|||
}
|
||||
|
||||
const last = new_candles[new_candles.len - 1];
|
||||
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
|
||||
var next = meta;
|
||||
next.last_close = last.close;
|
||||
next.last_date = last.date;
|
||||
self.updateCandleMeta(symbol, next, expires_at_s);
|
||||
}
|
||||
|
||||
/// Write (or refresh) candle metadata with a specific provider source.
|
||||
/// Write (or refresh) candle metadata.
|
||||
///
|
||||
/// `expires_at_s` is the absolute Unix-seconds freshness boundary for
|
||||
/// the `#!expires=` directive, computed by the caller via
|
||||
/// `market.nextCandleExpiry` (close-anchored) or a short retry. The
|
||||
/// cache layer no longer reads the wall clock for this - the boundary
|
||||
/// is market-domain knowledge owned by the caller.
|
||||
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||
const meta = CandleMeta{
|
||||
.last_close = last_close,
|
||||
.last_date = last_date,
|
||||
.provider = provider,
|
||||
.fail_count = fail_count,
|
||||
};
|
||||
///
|
||||
/// Takes the whole `CandleMeta` rather than a field-per-parameter
|
||||
/// list so that adding a metadata field does not churn every call
|
||||
/// site. Callers that mean "same metadata, one field changed"
|
||||
/// should copy the value they read and mutate the one field.
|
||||
pub fn updateCandleMeta(self: *Store, symbol: []const u8, meta: CandleMeta, expires_at_s: i64) void {
|
||||
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| {
|
||||
defer self.allocator.free(meta_data);
|
||||
self.writeRaw(symbol, .candles_meta, meta_data) catch |err| {
|
||||
|
|
@ -1348,27 +1388,118 @@ pub const Store = struct {
|
|||
pub const CandleMeta = struct {
|
||||
last_close: f64,
|
||||
last_date: Date,
|
||||
/// Which provider sourced the candle data. **No default
|
||||
/// value on purpose** - SRF auto-elides fields whose value
|
||||
/// equals their default, which would hide the provider line
|
||||
/// when it equaled the implicit default. We want every cache
|
||||
/// file to record its provider explicitly so cache inspection
|
||||
/// can always answer "where did this come from?". Construction
|
||||
/// sites must pass the provider explicitly.
|
||||
/// Which provider sourced the candle data. Pure provenance -
|
||||
/// this field answers "where did these bars come from?" and
|
||||
/// nothing else. It deliberately does **not** drive provider
|
||||
/// routing; see `tiingo_retry_after_s` for that. Conflating
|
||||
/// the two is what produced the one-way drift to Yahoo that
|
||||
/// this field's value used to cause.
|
||||
///
|
||||
/// **No default value on purpose** - SRF auto-elides fields
|
||||
/// whose value equals their default, which would hide the
|
||||
/// provider line when it equaled the implicit default. We
|
||||
/// want every cache file to record its provider explicitly
|
||||
/// so cache inspection can always answer "where did this
|
||||
/// come from?". Construction sites must pass the provider
|
||||
/// explicitly.
|
||||
///
|
||||
/// Cache compatibility: pre-2026-05 caches that elided the
|
||||
/// provider field will fail to deserialize after this change
|
||||
/// (SRF returns FieldNotFoundOnFieldWithoutDefaultValue).
|
||||
/// `readCandleMeta` swallows the error and returns null,
|
||||
/// making the symbol look like a cache miss - `getCandles`
|
||||
/// then triggers a fresh fetch via `populateAllFromTiingo`,
|
||||
/// then triggers a fresh fetch via `refetchFullHistory`,
|
||||
/// which writes a new meta file with the provider explicit.
|
||||
/// The wipe happens naturally on first use post-upgrade.
|
||||
///
|
||||
/// This is also why no *other* field on this struct may be
|
||||
/// default-less: that cold-start path is the destructive one.
|
||||
/// It writes a negative-cache marker over `candles_daily.srf`
|
||||
/// when no provider carries the symbol, so routing a
|
||||
/// populated cache through it risks discarding real history.
|
||||
/// Fields added since (`tiingo_retry_after_s`, `adj_basis`)
|
||||
/// are defaulted so legacy caches keep parsing.
|
||||
provider: CandleProvider,
|
||||
/// Consecutive transient failure count for the primary provider (Tiingo).
|
||||
/// Incremented on ServerError; reset to 0 on success. When >= 3, the
|
||||
/// symbol is degraded to a fallback provider until Tiingo recovers.
|
||||
/// Consecutive transient-failure count for a candle fetch
|
||||
/// (ServerError / connection failure). Reset to 0 on any
|
||||
/// successful fetch. At >= 3, `getCandles` serves the stale
|
||||
/// cached series rather than returning an error, so a provider
|
||||
/// outage degrades to "slightly old prices" instead of "no
|
||||
/// prices".
|
||||
///
|
||||
/// It does **not** switch providers. Provider selection is
|
||||
/// `tiingo_retry_after_s`'s job, and only a genuine 404 moves
|
||||
/// it - a transient outage says nothing about coverage.
|
||||
fail_count: u8 = 0,
|
||||
/// Unix-seconds instant before which Tiingo should not be
|
||||
/// consulted for this symbol. `0` (the default) means "no
|
||||
/// backoff - always try Tiingo first".
|
||||
///
|
||||
/// Set **only** when Tiingo returns a genuine 404, which is a
|
||||
/// fact about coverage rather than about the request. A 400 /
|
||||
/// 402 / malformed body falls back to Yahoo for that one call
|
||||
/// but leaves this field alone, so the next call retries
|
||||
/// Tiingo. Cleared back to `0` the moment Tiingo serves the
|
||||
/// symbol again.
|
||||
///
|
||||
/// Why this exists: routing used to key off `provider ==
|
||||
/// .yahoo`, which a single non-transient Tiingo failure would
|
||||
/// latch permanently - Yahoo was then tried first, succeeded,
|
||||
/// and rewrote `provider = .yahoo`, so Tiingo was never
|
||||
/// consulted again. An explicit, expiring signal makes the
|
||||
/// demotion recoverable and records *why* it happened.
|
||||
///
|
||||
/// Defaulted so legacy caches (which lack the field) parse
|
||||
/// cleanly and simply behave as "no backoff".
|
||||
tiingo_retry_after_s: i64 = 0,
|
||||
/// The bar date through which this series' `adj_close` values
|
||||
/// reflect corporate actions - i.e. the newest bar present at
|
||||
/// the last *full* fetch.
|
||||
///
|
||||
/// Providers compute `adj_close` by scaling raw `close` by the
|
||||
/// product of the adjustment factors for every distribution
|
||||
/// *after* that bar. So the whole series' adjustment basis is
|
||||
/// only as current as the fetch that produced it: a full fetch
|
||||
/// as of date D reflects every ex-date <= D, and nothing later.
|
||||
///
|
||||
/// Appends never advance this. `getCandles` tops the cache up
|
||||
/// incrementally, and the newly-appended bars arrive with
|
||||
/// `adj_close == close` (nothing has gone ex after them yet)
|
||||
/// while every previously-cached bar keeps its original basis.
|
||||
/// When a distribution later goes ex, the bars behind it should
|
||||
/// be marked down and nothing does it - so total returns read
|
||||
/// low by roughly the missed yield. Comparing this field
|
||||
/// against the newest known dividend/split ex-date is how
|
||||
/// `getCandles` detects that and escalates to a full refetch.
|
||||
///
|
||||
/// `Date.epoch` (the default) means "unknown - assume stale".
|
||||
/// Any real ex-date is later than it, so a dividend payer
|
||||
/// escalates on its first stale pass; a symbol with no
|
||||
/// corporate actions has no ex-date to exceed it and never
|
||||
/// escalates. Defaulted rather than required precisely so
|
||||
/// legacy caches keep parsing: making it required would route
|
||||
/// every cached symbol through the cold-start path, which is
|
||||
/// Tiingo-only and writes a negative-cache marker *over* the
|
||||
/// candle file on a 404 - destroying history for any symbol
|
||||
/// Tiingo does not carry.
|
||||
adj_basis: Date = Date.epoch,
|
||||
};
|
||||
|
||||
/// The subset of `CandleMeta` that describes provider state rather
|
||||
/// than the candle data itself.
|
||||
///
|
||||
/// `cacheCandles` takes this instead of a whole `CandleMeta`
|
||||
/// because a full fetch may be a cold start with no prior metadata
|
||||
/// to copy - there is no honest `last_close` / `last_date` for the
|
||||
/// caller to supply, and those get derived from the candles being
|
||||
/// written anyway. `appendCandles` by contrast always has the
|
||||
/// existing meta in hand (it read `last_date` to decide what to
|
||||
/// fetch), so it takes the full struct and preserves everything it
|
||||
/// does not derive.
|
||||
pub const CandleMetaAttrs = struct {
|
||||
provider: CandleProvider,
|
||||
fail_count: u8 = 0,
|
||||
tiingo_retry_after_s: i64 = 0,
|
||||
};
|
||||
|
||||
pub const CandleProvider = enum {
|
||||
|
|
@ -1376,17 +1507,22 @@ pub const Store = struct {
|
|||
/// writes produce this value (TwelveData was demoted in an
|
||||
/// earlier change because its `adj_close` was unreliable).
|
||||
/// Cache reads still recognize the value for backwards
|
||||
/// compatibility.
|
||||
/// compatibility, and `getCandles` treats such a cache as
|
||||
/// unusable so the symbol gets refetched.
|
||||
twelvedata,
|
||||
/// Legacy: candles were sourced from Yahoo Finance. No new
|
||||
/// writes produce this value (Yahoo was removed from the
|
||||
/// candle pipeline in the 2026-05 audit; Yahoo is still used
|
||||
/// for `getQuote` real-time prices but not for historical
|
||||
/// candles). Cache reads still recognize the value for
|
||||
/// backwards compatibility.
|
||||
/// Candles were sourced from Yahoo Finance.
|
||||
///
|
||||
/// Actively written. `fetchCandlesFromProviders` falls back to
|
||||
/// Yahoo whenever Tiingo cannot serve a symbol, and
|
||||
/// `refetchFullHistory` does the same for a full restatement.
|
||||
/// Yahoo's `adj_close` is split- and dividend-adjusted like
|
||||
/// Tiingo's, but its parser yields 0 for a JSON `null` element,
|
||||
/// which the analytics layer then has to discard - so Tiingo is
|
||||
/// preferred where both will answer.
|
||||
yahoo,
|
||||
/// Active: candles sourced from Tiingo. The only value
|
||||
/// produced by current writes.
|
||||
/// Candles were sourced from Tiingo. Preferred: it serves
|
||||
/// dividends and splits from the same response, and its
|
||||
/// `adj_close` is what the analytics layer is written against.
|
||||
tiingo,
|
||||
|
||||
pub fn fromString(s: []const u8) CandleProvider {
|
||||
|
|
@ -3568,7 +3704,7 @@ test "deserializeCandleMeta fails on old cache that elided provider field" {
|
|||
// (model has no default for provider). The graceful handling is
|
||||
// upstream: `readCandleMeta` swallows the deserialization error
|
||||
// and returns null, which makes `getCandles` treat it as a cache
|
||||
// miss and trigger a fresh fetch via `populateAllFromTiingo`.
|
||||
// miss and trigger a fresh fetch via `refetchFullHistory`.
|
||||
// The new fetch writes a meta file with the provider explicit.
|
||||
//
|
||||
// This test documents the failure mode and confirms it's not a
|
||||
|
|
@ -3585,6 +3721,214 @@ test "deserializeCandleMeta fails on old cache that elided provider field" {
|
|||
try std.testing.expectError(error.InvalidData, result);
|
||||
}
|
||||
|
||||
test "CandleMeta.tiingo_retry_after_s defaults to 0 and is elided when unset" {
|
||||
// The field is defaulted precisely so caches written before it
|
||||
// existed keep parsing. SRF elides default-valued fields, so an
|
||||
// unset backoff costs no bytes and reads back as "no backoff".
|
||||
const allocator = std.testing.allocator;
|
||||
const meta = Store.CandleMeta{
|
||||
.last_close = 100.0,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
try std.testing.expectEqual(@as(i64, 0), meta.tiingo_retry_after_s);
|
||||
|
||||
const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 });
|
||||
defer allocator.free(data);
|
||||
try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s") == null);
|
||||
}
|
||||
|
||||
test "CandleMeta.tiingo_retry_after_s round-trips when armed" {
|
||||
const allocator = std.testing.allocator;
|
||||
const meta = Store.CandleMeta{
|
||||
.last_close = 100.0,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .yahoo,
|
||||
.tiingo_retry_after_s = 1_790_000_000,
|
||||
};
|
||||
const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 });
|
||||
defer allocator.free(data);
|
||||
try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s:num:1790000000") != null);
|
||||
|
||||
const parsed = try Store.deserializeCandleMeta(allocator, data);
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), parsed.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider);
|
||||
}
|
||||
|
||||
test "legacy candles_meta without tiingo_retry_after_s still parses" {
|
||||
// The whole migration story for this field: unlike `provider`
|
||||
// (which is deliberately default-less and therefore wipes old
|
||||
// caches), a defaulted field must NOT break existing caches. If
|
||||
// this ever regresses, every cached symbol takes a full re-fetch
|
||||
// on first use - and for symbols Tiingo does not carry, the
|
||||
// cold-start path would write a negative-cache marker over a
|
||||
// perfectly good candle history.
|
||||
const allocator = std.testing.allocator;
|
||||
const legacy =
|
||||
\\#!srfv1
|
||||
\\#!expires=1787000100
|
||||
\\#!created=1786748010
|
||||
\\last_close:num:776.34,last_date::2026-08-14,provider::yahoo
|
||||
\\
|
||||
;
|
||||
const parsed = try Store.deserializeCandleMeta(allocator, legacy);
|
||||
try std.testing.expectEqual(@as(i64, 0), parsed.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider);
|
||||
try std.testing.expect(parsed.last_date.eql(Date.fromYmd(2026, 8, 14)));
|
||||
}
|
||||
|
||||
test "appendCandles preserves caller-supplied provider state" {
|
||||
// appendCandles derives only last_close/last_date; every other
|
||||
// metadata field must pass through untouched. Commit-2's
|
||||
// `adj_basis` depends on this exact property, so pin it now.
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
var store = Store.init(io, allocator, dir_path);
|
||||
const seed = [_]Candle{.{
|
||||
.date = Date.fromYmd(2026, 8, 13),
|
||||
.open = 1,
|
||||
.high = 1,
|
||||
.low = 1,
|
||||
.close = 1,
|
||||
.adj_close = 1,
|
||||
.volume = 1,
|
||||
}};
|
||||
store.cacheCandles("SMPL", seed[0..], .{ .provider = .yahoo, .tiingo_retry_after_s = 1_790_000_000 }, 9_999_999_999);
|
||||
|
||||
const before = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult;
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), before.meta.tiingo_retry_after_s);
|
||||
|
||||
const more = [_]Candle{.{
|
||||
.date = Date.fromYmd(2026, 8, 14),
|
||||
.open = 2,
|
||||
.high = 2,
|
||||
.low = 2,
|
||||
.close = 2,
|
||||
.adj_close = 2,
|
||||
.volume = 2,
|
||||
}};
|
||||
store.appendCandles("SMPL", more[0..], before.meta, 9_999_999_999);
|
||||
|
||||
const after = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult;
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), after.meta.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(Store.CandleProvider.yahoo, after.meta.provider);
|
||||
// ...while the derived fields did advance.
|
||||
try std.testing.expect(after.meta.last_date.eql(Date.fromYmd(2026, 8, 14)));
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 2), after.meta.last_close, 0.001);
|
||||
}
|
||||
|
||||
// ── adj_basis: adjustment-basis staleness ────────────────────
|
||||
//
|
||||
// The cache is append-only. A distribution that goes ex after the last
|
||||
// full fetch never marks down the bars behind it, so total returns read
|
||||
// low by roughly the missed yield. `adj_basis` records how current the
|
||||
// series' adjustment basis is; these tests pin the detection.
|
||||
|
||||
/// Build a one-bar-per-day series ending at `last`, for basis tests.
|
||||
fn basisTestCandles(buf: []Candle, first: Date, count: usize) []Candle {
|
||||
for (0..count) |i| {
|
||||
const d = first.addDays(@intCast(i));
|
||||
buf[i] = .{ .date = d, .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 };
|
||||
}
|
||||
return buf[0..count];
|
||||
}
|
||||
|
||||
test "cacheCandles stamps adj_basis at the newest bar" {
|
||||
// A full replace means every row carries the provider's current
|
||||
// adjustment basis, which by construction covers every corporate
|
||||
// action up to the newest bar.
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
var s = Store.init(io, allocator, dir_path);
|
||||
var buf: [5]Candle = undefined;
|
||||
const candles = basisTestCandles(&buf, Date.fromYmd(2026, 8, 10), 5);
|
||||
s.cacheCandles("SMPL", candles, .{ .provider = .tiingo }, 9_999_999_999);
|
||||
|
||||
const meta = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
|
||||
try std.testing.expect(meta.adj_basis.eql(Date.fromYmd(2026, 8, 14)));
|
||||
try std.testing.expect(meta.last_date.eql(Date.fromYmd(2026, 8, 14)));
|
||||
}
|
||||
|
||||
test "appendCandles does not advance adj_basis" {
|
||||
// The core invariant. Appended bars arrive with adj_close == close
|
||||
// (nothing has gone ex after them yet) and do NOT restate the rows
|
||||
// already on disk, so claiming a newer basis would mask exactly the
|
||||
// staleness this field exists to detect.
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
var s = Store.init(io, allocator, dir_path);
|
||||
var buf: [3]Candle = undefined;
|
||||
s.cacheCandles("SMPL", basisTestCandles(&buf, Date.fromYmd(2026, 6, 1), 3), .{ .provider = .tiingo }, 9_999_999_999);
|
||||
const before = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
|
||||
try std.testing.expect(before.adj_basis.eql(Date.fromYmd(2026, 6, 3)));
|
||||
|
||||
var more: [3]Candle = undefined;
|
||||
s.appendCandles("SMPL", basisTestCandles(&more, Date.fromYmd(2026, 8, 12), 3), before, 9_999_999_999);
|
||||
|
||||
const after = (s.readCandleMeta("SMPL") orelse return error.NoCache).meta;
|
||||
// last_date advanced...
|
||||
try std.testing.expect(after.last_date.eql(Date.fromYmd(2026, 8, 14)));
|
||||
// ...but the basis is pinned to the last full fetch.
|
||||
try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 3)));
|
||||
}
|
||||
|
||||
test "legacy candles_meta without adj_basis parses to the epoch sentinel" {
|
||||
// Same migration story as tiingo_retry_after_s, and the stakes are
|
||||
// higher: making this field required would route every cached symbol
|
||||
// through the cold-start path, which on a 404 writes a negative
|
||||
// marker OVER candles_daily.srf and destroys the history.
|
||||
const allocator = std.testing.allocator;
|
||||
const legacy =
|
||||
\\#!srfv1
|
||||
\\#!expires=1787000100
|
||||
\\#!created=1786748010
|
||||
\\last_close:num:776.34,last_date::2026-08-14,provider::yahoo
|
||||
\\
|
||||
;
|
||||
const parsed = try Store.deserializeCandleMeta(allocator, legacy);
|
||||
try std.testing.expect(parsed.adj_basis.eql(Date.epoch));
|
||||
// A sentinel basis on a dividend payer is exactly the "needs
|
||||
// restatement" signal, so it must not be mistaken for "current".
|
||||
try std.testing.expect(parsed.adj_basis.lessThan(Date.fromYmd(2026, 6, 18)));
|
||||
}
|
||||
|
||||
test "adj_basis is elided when unset and round-trips when set" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
const unset = Store.CandleMeta{
|
||||
.last_close = 100.0,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
const bare = try Store.serializeCandleMeta(std.testing.io, allocator, unset, .{ .expires = 1 });
|
||||
defer allocator.free(bare);
|
||||
try std.testing.expect(std.mem.indexOf(u8, bare, "adj_basis") == null);
|
||||
|
||||
var set = unset;
|
||||
set.adj_basis = Date.fromYmd(2026, 8, 14);
|
||||
const full = try Store.serializeCandleMeta(std.testing.io, allocator, set, .{ .expires = 1 });
|
||||
defer allocator.free(full);
|
||||
try std.testing.expect(std.mem.indexOf(u8, full, "adj_basis::2026-08-14") != null);
|
||||
|
||||
const parsed = try Store.deserializeCandleMeta(allocator, full);
|
||||
try std.testing.expect(parsed.adj_basis.eql(Date.fromYmd(2026, 8, 14)));
|
||||
}
|
||||
|
||||
// ── writeRaw / appendRaw atomicity ───────────────────────────
|
||||
//
|
||||
// A concurrent reader hitting a cache file mid-write must never see a
|
||||
|
|
|
|||
|
|
@ -239,7 +239,36 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
|
|||
try out.print(", {s}", .{if (obs.local_fresh) "TTL still in the future" else "TTL lapsed"});
|
||||
try out.print(", {s}", .{@tagName(m.meta.provider)});
|
||||
if (m.meta.fail_count > 0) try out.print(", {d} consecutive failures", .{m.meta.fail_count});
|
||||
// A live Tiingo backoff means this symbol is being served by
|
||||
// Yahoo on purpose (Tiingo 404'd it), not by accident. Surface
|
||||
// it so a provider that looks "wrong" can be explained.
|
||||
if (m.meta.tiingo_retry_after_s > ctx.now_s) {
|
||||
try out.print(", Tiingo backoff until {f}", .{Date.fromEpoch(m.meta.tiingo_retry_after_s)});
|
||||
} else if (m.meta.tiingo_retry_after_s != 0) {
|
||||
try out.print(", Tiingo backoff lapsed (retries next refresh)", .{});
|
||||
}
|
||||
try out.print("\n", .{});
|
||||
|
||||
// Adjustment basis. The cache is append-only, so a distribution
|
||||
// that goes ex after the last full fetch never marks down the
|
||||
// bars behind it - total returns then read low by roughly the
|
||||
// missed yield. Report it here because the symptom (a slightly
|
||||
// low 1Y total return) is otherwise invisible.
|
||||
if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| {
|
||||
if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) {
|
||||
try out.print(
|
||||
"adj basis {f} - STALE, {f} went ex behind it; total returns read low until restated\n",
|
||||
.{ m.meta.adj_basis, newest_action },
|
||||
);
|
||||
} else {
|
||||
try out.print(
|
||||
"adj basis {f} - current through the newest corporate action ({f})\n",
|
||||
.{ m.meta.adj_basis, newest_action },
|
||||
);
|
||||
}
|
||||
} else {
|
||||
try out.print("adj basis {f} - no dividends or splits cached, nothing to restate\n", .{m.meta.adj_basis});
|
||||
}
|
||||
} else {
|
||||
try out.print("local nothing cached\n", .{});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,20 +83,21 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
|
|||
try out.print(")\nLatest close: {f}\n", .{Money.from(c[c.len - 1].close)});
|
||||
|
||||
// `dividends != null` indicates we got explicit dividend records
|
||||
// from the provider. When false we still display total return
|
||||
// (synthesized from adj_close, which most providers bake dividends
|
||||
// into), but we surface a hint that explicit dividend data is
|
||||
// missing.
|
||||
// from the provider. When false we still display total return -
|
||||
// `performance.totalReturns` degrades to the provider's `adj_close`,
|
||||
// which is dividend-adjusted - but we surface a hint, because that
|
||||
// series is only as current as the last full candle fetch (see
|
||||
// `CandleMeta.adj_basis`) and understates by the missed yield when a
|
||||
// distribution has gone ex since.
|
||||
const has_explicit_divs = result.dividends != null;
|
||||
const has_total = result.asof_total != null;
|
||||
|
||||
// -- As-of-date returns --
|
||||
try cli.printBold(out, color, "\nAs-of {f}:\n", .{end_date});
|
||||
try printReturnsTable(out, result.asof_price, if (has_total) result.asof_total else null, color);
|
||||
try printReturnsTable(out, result.asof_price, result.asof_total, color);
|
||||
|
||||
// -- Month-end returns --
|
||||
try cli.printBold(out, color, "\nMonth-end ({f}):\n", .{month_end});
|
||||
try printReturnsTable(out, result.me_price, if (has_total) result.me_total else null, color);
|
||||
try printReturnsTable(out, result.me_price, result.me_total, color);
|
||||
|
||||
if (!has_explicit_divs) {
|
||||
try cli.printFg(out, color, cli.CLR_MUTED, "\nSet POLYGON_API_KEY for total returns with dividend reinvestment.\n", .{});
|
||||
|
|
|
|||
|
|
@ -220,12 +220,32 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
|
|||
}
|
||||
}
|
||||
|
||||
// Per-symbol splits. Fetched alongside dividends because the totals
|
||||
// row's total-return index is built from raw `close`, which is not
|
||||
// split-adjusted - dividends without splits would read a 2:1 split
|
||||
// as a -50% month. Splits ride along in the same Tiingo candle
|
||||
// response the dividend warm above already paid for.
|
||||
var split_map = std.StringHashMap([]const zfin.Split).init(allocator);
|
||||
defer {
|
||||
var it = split_map.iterator();
|
||||
while (it.next()) |entry| {
|
||||
allocator.free(@constCast(entry.value_ptr.*));
|
||||
}
|
||||
split_map.deinit();
|
||||
}
|
||||
for (pf_data.summary.allocations) |a| {
|
||||
if (svc.getCachedSplits(allocator, a.symbol)) |spl| {
|
||||
try split_map.put(a.symbol, spl.data);
|
||||
}
|
||||
}
|
||||
|
||||
var view = try review_view.buildReview(
|
||||
allocator,
|
||||
io,
|
||||
pf_data.summary,
|
||||
&pf_data.candle_map,
|
||||
÷nd_map,
|
||||
&split_map,
|
||||
portfolio,
|
||||
cm,
|
||||
acct_map_opt,
|
||||
|
|
|
|||
107
src/history.zig
107
src/history.zig
|
|
@ -34,7 +34,6 @@ const builtin = @import("builtin");
|
|||
const srf = @import("srf");
|
||||
const snapshot = @import("models/snapshot.zig");
|
||||
const Date = @import("Date.zig");
|
||||
const Candle = @import("models/candle.zig").Candle;
|
||||
const timeline = @import("analytics/timeline.zig");
|
||||
const valuation = @import("analytics/valuation.zig");
|
||||
const imported_values = @import("data/imported_values.zig");
|
||||
|
|
@ -564,35 +563,6 @@ pub fn resolveSnapshotDate(
|
|||
|
||||
// ── Pure-domain aggregation ───────────────────────────────────
|
||||
|
||||
/// Return the prefix of `candles` whose dates are `<= as_of`.
|
||||
///
|
||||
/// When `as_of` is null, returns the full slice unchanged (live mode
|
||||
/// pass-through). When set, binary-searches for the first index
|
||||
/// strictly after `as_of` and slices up to it. Zero-length slice
|
||||
/// when `as_of` precedes all cached candles.
|
||||
///
|
||||
/// Candles are assumed sorted by date ascending. Used to truncate
|
||||
/// benchmark and per-symbol price history for historical projections -
|
||||
/// `performance.trailingReturns` uses the last candle's date as the
|
||||
/// endpoint, so trimming the tail is equivalent to "compute as of
|
||||
/// that date".
|
||||
pub fn sliceCandlesAsOf(candles: []const Candle, as_of: ?Date) []const Candle {
|
||||
const d = as_of orelse return candles;
|
||||
if (candles.len == 0) return candles;
|
||||
var lo: usize = 0;
|
||||
var hi: usize = candles.len;
|
||||
while (lo < hi) {
|
||||
const mid = lo + (hi - lo) / 2;
|
||||
const cd = candles[mid].date;
|
||||
if (cd.lessThan(d) or cd.eql(d)) {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
return candles[0..lo];
|
||||
}
|
||||
|
||||
/// Find the `scope=="liquid"` total in a snapshot. Returns 0.0 if not
|
||||
/// present (old snapshots from before the liquid/illiquid split -
|
||||
/// shouldn't happen in practice).
|
||||
|
|
@ -1344,83 +1314,6 @@ test "aggregateSnapshotAllocations: aggregates by `symbol` (pricing), not `lot_s
|
|||
try testing.expectApproxEqAbs(@as(f64, 6_750), sa.allocations[0].market_value, 0.01);
|
||||
}
|
||||
|
||||
// ── sliceCandlesAsOf tests ────────────────────────────────────
|
||||
|
||||
fn makeTestCandle(y: i16, m: u8, d: u8, close: f64) Candle {
|
||||
return .{
|
||||
.date = Date.fromYmd(y, m, d),
|
||||
.open = close,
|
||||
.high = close,
|
||||
.low = close,
|
||||
.close = close,
|
||||
.adj_close = close,
|
||||
.volume = 0,
|
||||
};
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: null as_of returns everything" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, null);
|
||||
try testing.expectEqual(@as(usize, 2), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: empty input" {
|
||||
const candles = [_]Candle{};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 1));
|
||||
try testing.expectEqual(@as(usize, 0), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: empty input with null as_of" {
|
||||
const candles = [_]Candle{};
|
||||
const sliced = sliceCandlesAsOf(&candles, null);
|
||||
try testing.expectEqual(@as(usize, 0), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: exact date match included" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
makeTestCandle(2024, 1, 3, 102),
|
||||
makeTestCandle(2024, 1, 4, 103),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
|
||||
try testing.expectEqual(@as(usize, 2), sliced.len);
|
||||
try testing.expectApproxEqAbs(@as(f64, 101), sliced[sliced.len - 1].close, 0.001);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: no exact match snaps to earlier" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 3, 102), // gap - no candle on the 2nd
|
||||
makeTestCandle(2024, 1, 4, 103),
|
||||
};
|
||||
// Asking for Jan 2 returns everything through Jan 1 (nothing at/after Jan 2).
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
|
||||
try testing.expectEqual(@as(usize, 1), sliced.len);
|
||||
try testing.expectApproxEqAbs(@as(f64, 100), sliced[0].close, 0.001);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: as_of before all candles returns empty" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2023, 12, 31));
|
||||
try testing.expectEqual(@as(usize, 0), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: as_of after all candles returns everything" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2026, 1, 1));
|
||||
try testing.expectEqual(@as(usize, 2), sliced.len);
|
||||
}
|
||||
|
||||
// ── resolveSnapshotDate tests ─────────────────────────────────
|
||||
|
||||
test "resolveSnapshotDate: exact match returns exact=true" {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
const std = @import("std");
|
||||
const testing = std.testing;
|
||||
const Date = @import("../Date.zig");
|
||||
|
||||
/// A single OHLCV bar, normalized from any provider.
|
||||
|
|
@ -21,13 +23,47 @@ pub const Candle = struct {
|
|||
/// older cached candles or providers that don't populate
|
||||
/// `adj_close` leave it at 0; in that case the raw close is
|
||||
/// the best we can do.
|
||||
///
|
||||
/// The zero fallback matters beyond charts: Yahoo's parser yields
|
||||
/// 0 for a JSON `null` element, and a risk series that divides by
|
||||
/// it drops the month entirely. `analytics/portfolio_risk.zig`
|
||||
/// reuses this for that reason - "adjusted close, or raw close
|
||||
/// when the provider left it unusable" is the same question there.
|
||||
pub fn chartClose(self: Candle) f64 {
|
||||
return if (self.adj_close != 0) self.adj_close else self.close;
|
||||
}
|
||||
};
|
||||
|
||||
/// Return the prefix of `candles` whose dates are `<= as_of`.
|
||||
///
|
||||
/// When `as_of` is null, returns the full slice unchanged (live mode
|
||||
/// pass-through). When set, binary-searches for the first index
|
||||
/// strictly after `as_of` and slices up to it. Zero-length slice
|
||||
/// when `as_of` precedes all cached candles.
|
||||
///
|
||||
/// Candles are assumed sorted by date ascending. Truncating the tail is
|
||||
/// how "compute as of date D" is expressed throughout the analytics
|
||||
/// layer: every window that derives its endpoint from the newest bar
|
||||
/// (trailing returns, week change, month-end snaps) then lands on D
|
||||
/// without needing its own as-of parameter.
|
||||
pub fn sliceCandlesAsOf(candles: []const Candle, as_of: ?Date) []const Candle {
|
||||
const d = as_of orelse return candles;
|
||||
if (candles.len == 0) return candles;
|
||||
var lo: usize = 0;
|
||||
var hi: usize = candles.len;
|
||||
while (lo < hi) {
|
||||
const mid = lo + (hi - lo) / 2;
|
||||
const cd = candles[mid].date;
|
||||
if (cd.lessThan(d) or cd.eql(d)) {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
return candles[0..lo];
|
||||
}
|
||||
|
||||
test "chartClose prefers adj_close when populated" {
|
||||
const std = @import("std");
|
||||
const c = Candle{
|
||||
.date = Date.fromYmd(2024, 3, 7),
|
||||
.open = 700.0,
|
||||
|
|
@ -41,7 +77,6 @@ test "chartClose prefers adj_close when populated" {
|
|||
}
|
||||
|
||||
test "chartClose falls back to close when adj_close is zero" {
|
||||
const std = @import("std");
|
||||
const c = Candle{
|
||||
.date = Date.fromYmd(2024, 1, 1),
|
||||
.open = 100.0,
|
||||
|
|
@ -55,7 +90,6 @@ test "chartClose falls back to close when adj_close is zero" {
|
|||
}
|
||||
|
||||
test "chartClose synthetic split has continuous chart values" {
|
||||
const std = @import("std");
|
||||
// Pre-split: close=300, adj_close=100 (after a 3:1 split)
|
||||
// Post-split: close=100, adj_close=100
|
||||
// Chart should see [100, 100] not [300, 100].
|
||||
|
|
@ -79,3 +113,80 @@ test "chartClose synthetic split has continuous chart values" {
|
|||
};
|
||||
try std.testing.expectEqual(pre.chartClose(), post.chartClose());
|
||||
}
|
||||
|
||||
// ── sliceCandlesAsOf tests ────────────────────────────────────
|
||||
|
||||
fn makeTestCandle(y: i16, m: u8, d: u8, close: f64) Candle {
|
||||
return .{
|
||||
.date = Date.fromYmd(y, m, d),
|
||||
.open = close,
|
||||
.high = close,
|
||||
.low = close,
|
||||
.close = close,
|
||||
.adj_close = close,
|
||||
.volume = 0,
|
||||
};
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: null as_of returns everything" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, null);
|
||||
try testing.expectEqual(@as(usize, 2), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: empty input" {
|
||||
const candles = [_]Candle{};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 1));
|
||||
try testing.expectEqual(@as(usize, 0), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: empty input with null as_of" {
|
||||
const candles = [_]Candle{};
|
||||
const sliced = sliceCandlesAsOf(&candles, null);
|
||||
try testing.expectEqual(@as(usize, 0), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: exact date match included" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
makeTestCandle(2024, 1, 3, 102),
|
||||
makeTestCandle(2024, 1, 4, 103),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
|
||||
try testing.expectEqual(@as(usize, 2), sliced.len);
|
||||
try testing.expectApproxEqAbs(@as(f64, 101), sliced[sliced.len - 1].close, 0.001);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: no exact match snaps to earlier" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 3, 102), // gap - no candle on the 2nd
|
||||
makeTestCandle(2024, 1, 4, 103),
|
||||
};
|
||||
// Asking for Jan 2 returns everything through Jan 1 (nothing at/after Jan 2).
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2024, 1, 2));
|
||||
try testing.expectEqual(@as(usize, 1), sliced.len);
|
||||
try testing.expectApproxEqAbs(@as(f64, 100), sliced[0].close, 0.001);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: as_of before all candles returns empty" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2023, 12, 31));
|
||||
try testing.expectEqual(@as(usize, 0), sliced.len);
|
||||
}
|
||||
|
||||
test "sliceCandlesAsOf: as_of after all candles returns everything" {
|
||||
const candles = [_]Candle{
|
||||
makeTestCandle(2024, 1, 1, 100),
|
||||
makeTestCandle(2024, 1, 2, 101),
|
||||
};
|
||||
const sliced = sliceCandlesAsOf(&candles, Date.fromYmd(2026, 1, 1));
|
||||
try testing.expectEqual(@as(usize, 2), sliced.len);
|
||||
}
|
||||
|
|
|
|||
235
src/net/http.zig
235
src/net/http.zig
|
|
@ -250,7 +250,7 @@ pub const Client = struct {
|
|||
}.f;
|
||||
|
||||
const uri = std.Uri.parse(url) catch |err| {
|
||||
log.warn("http {s}: stage=uri_parse err={s} url={s}", .{ @tagName(method), @errorName(err), url });
|
||||
log.warn("http {s}: stage=uri_parse err={s} url={f}", .{ @tagName(method), @errorName(err), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
const ms_uri_parse = stageElapsedMs(&t_stage, self.io);
|
||||
|
|
@ -284,7 +284,7 @@ pub const Client = struct {
|
|||
// TLS handshake. Logging at warn level (rather than debug)
|
||||
// because DNS / connectivity failures are exactly what
|
||||
// operators need to see immediately.
|
||||
log.warn("http {s}: stage=connect err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=connect err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
defer req.deinit();
|
||||
|
|
@ -294,24 +294,24 @@ pub const Client = struct {
|
|||
var send_buf: [4096]u8 = undefined;
|
||||
req.transfer_encoding = .{ .content_length = payload.len };
|
||||
var bw = req.sendBodyUnflushed(&send_buf) catch |err| {
|
||||
log.warn("http {s}: stage=send_body_open err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=send_body_open err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
bw.writer.writeAll(payload) catch |err| {
|
||||
log.warn("http {s}: stage=send_body_write err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=send_body_write err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
bw.end() catch |err| {
|
||||
log.warn("http {s}: stage=send_body_end err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=send_body_end err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
req.connection.?.flush() catch |err| {
|
||||
log.warn("http {s}: stage=send_body_flush err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=send_body_flush err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
} else {
|
||||
req.sendBodiless() catch |err| {
|
||||
log.warn("http {s}: stage=send_bodiless err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=send_bodiless err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
}
|
||||
|
|
@ -320,7 +320,7 @@ pub const Client = struct {
|
|||
// Matches the default redirect capacity in std.http.Client.fetch.
|
||||
var redirect_buffer: [8 * 1024]u8 = undefined;
|
||||
var response = req.receiveHead(&redirect_buffer) catch |err| {
|
||||
log.warn("http {s}: stage=receive_head err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=receive_head err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
const ms_receive_head = stageElapsedMs(&t_stage, self.io);
|
||||
|
|
@ -359,7 +359,7 @@ pub const Client = struct {
|
|||
var decompress_buffer: [64 * 1024]u8 = undefined;
|
||||
const reader = response.readerDecompressing(&transfer_buffer, &decompress, &decompress_buffer);
|
||||
_ = reader.streamRemaining(&aw.writer) catch |err| {
|
||||
log.warn("http {s}: stage=stream_body err={s} elapsed_ms={d} url={s}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), url });
|
||||
log.warn("http {s}: stage=stream_body err={s} elapsed_ms={d} url={f}", .{ @tagName(method), @errorName(err), stageElapsedMs(&t_stage, self.io), redactUrl(url) });
|
||||
return err;
|
||||
};
|
||||
const ms_body = stageElapsedMs(&t_stage, self.io);
|
||||
|
|
@ -368,7 +368,7 @@ pub const Client = struct {
|
|||
|
||||
const total_ms = @divTrunc(std.Io.Timestamp.now(self.io, .awake).nanoseconds - t_start, std.time.ns_per_ms);
|
||||
log.debug(
|
||||
"http {s}: ok status={d} bytes={d} total_ms={d} (uri_parse={d} connect={d} send={d} receive_head={d} body={d}) url={s}",
|
||||
"http {s}: ok status={d} bytes={d} total_ms={d} (uri_parse={d} connect={d} send={d} receive_head={d} body={d}) url={f}",
|
||||
.{
|
||||
@tagName(method),
|
||||
@intFromEnum(response.head.status),
|
||||
|
|
@ -379,7 +379,7 @@ pub const Client = struct {
|
|||
ms_send,
|
||||
ms_receive_head,
|
||||
ms_body,
|
||||
url,
|
||||
redactUrl(url),
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -443,6 +443,117 @@ pub const Client = struct {
|
|||
}
|
||||
};
|
||||
|
||||
// ── URL redaction for logs ───────────────────────────────────
|
||||
//
|
||||
// Providers authenticate by query parameter, so a raw request URL is a
|
||||
// credential. Every log line here used to print `url={s}` verbatim -
|
||||
// including at `warn` level, which reaches production (ReleaseSafe
|
||||
// defaults to `.info`). A single upstream 5xx was therefore enough to
|
||||
// write a live API key into a cron log. Log through `redactUrl` and
|
||||
// `{f}` instead; never interpolate a URL with `{s}`.
|
||||
|
||||
/// Placeholder substituted for a credential value.
|
||||
const redacted_marker = "REDACTED";
|
||||
|
||||
/// Query-parameter names whose values are credentials.
|
||||
///
|
||||
/// Covers every in-tree provider (`token` for Tiingo, `apikey` for FMP
|
||||
/// and TwelveData, `apiKey` for Polygon) plus the common variants a
|
||||
/// future provider is likely to use. Matched case-insensitively.
|
||||
///
|
||||
/// **Adding a provider that authenticates by query parameter?** Add its
|
||||
/// parameter name here. `looksLikeCredential` is a backstop for the
|
||||
/// case where someone forgets, not a substitute for this list.
|
||||
const credential_params = [_][]const u8{
|
||||
"token",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"apitoken",
|
||||
"api_token",
|
||||
"key",
|
||||
"access_token",
|
||||
"auth",
|
||||
"secret",
|
||||
"password",
|
||||
};
|
||||
|
||||
fn isCredentialParam(name: []const u8) bool {
|
||||
for (credential_params) |candidate| {
|
||||
if (std.ascii.eqlIgnoreCase(name, candidate)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Backstop heuristic: does this value look like an opaque secret?
|
||||
///
|
||||
/// True for a long run of credential-alphabet bytes and nothing else.
|
||||
/// Every legitimate query value these providers send is either short
|
||||
/// (symbols, intervals, booleans, CIKs, CUSIPs - all under 20 bytes) or
|
||||
/// contains punctuation the alphabet below excludes (dates carry `-`
|
||||
/// but are 10 bytes; symbol lists carry `,`). Real keys are 32-40 bytes
|
||||
/// of hex or base62.
|
||||
///
|
||||
/// Deliberately biased toward over-redaction: a redacted debug line is
|
||||
/// an inconvenience, a leaked key is an incident.
|
||||
fn looksLikeCredential(value: []const u8) bool {
|
||||
if (value.len < 20) return false;
|
||||
for (value) |c| {
|
||||
switch (c) {
|
||||
'A'...'Z', 'a'...'z', '0'...'9', '_', '-' => {},
|
||||
else => return false,
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// A URL wrapped for safe logging. Render with `{f}`.
|
||||
pub const RedactedUrl = struct {
|
||||
url: []const u8,
|
||||
|
||||
pub fn format(self: RedactedUrl, w: *std.Io.Writer) std.Io.Writer.Error!void {
|
||||
// Everything up to '?' is scheme/host/path - no credentials.
|
||||
const q = std.mem.indexOfScalar(u8, self.url, '?') orelse {
|
||||
try w.writeAll(self.url);
|
||||
return;
|
||||
};
|
||||
try w.writeAll(self.url[0 .. q + 1]);
|
||||
|
||||
// `buildUrl` percent-encodes '&' and '=' out of values, so
|
||||
// splitting on them is unambiguous. Polygon hand-rolls its
|
||||
// `apiKey=` suffix but uses the same delimiters.
|
||||
var first = true;
|
||||
var it = std.mem.splitScalar(u8, self.url[q + 1 ..], '&');
|
||||
while (it.next()) |pair| {
|
||||
if (!first) try w.writeByte('&');
|
||||
first = false;
|
||||
|
||||
const eq = std.mem.indexOfScalar(u8, pair, '=') orelse {
|
||||
try w.writeAll(pair);
|
||||
continue;
|
||||
};
|
||||
const name = pair[0..eq];
|
||||
const value = pair[eq + 1 ..];
|
||||
|
||||
try w.writeAll(name);
|
||||
try w.writeByte('=');
|
||||
if (isCredentialParam(name) or looksLikeCredential(value)) {
|
||||
try w.writeAll(redacted_marker);
|
||||
} else {
|
||||
try w.writeAll(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Wrap a URL so `{f}` renders it with credential values elided.
|
||||
///
|
||||
/// Use this at every log site that reports a URL. The parameter name
|
||||
/// and the non-secret values survive, so an operator can still see
|
||||
/// which symbol and date range failed.
|
||||
pub fn redactUrl(url: []const u8) RedactedUrl {
|
||||
return .{ .url = url };
|
||||
}
|
||||
|
||||
/// Build a URL with query parameters. Values are percent-encoded per RFC 3986.
|
||||
pub fn buildUrl(
|
||||
allocator: std.mem.Allocator,
|
||||
|
|
@ -636,3 +747,105 @@ test "Response.verifyIntegrity: mismatched sha256 returns mismatch" {
|
|||
else => try std.testing.expect(false),
|
||||
}
|
||||
}
|
||||
|
||||
// ── redactUrl ────────────────────────────────────────────────
|
||||
//
|
||||
// Regression suite for a credential leak: every `url=` log line - nine
|
||||
// of them at `warn`, which reaches production under ReleaseSafe - used
|
||||
// to print the request URL verbatim, so any upstream failure wrote a
|
||||
// live API key into the log.
|
||||
|
||||
/// Render through the `{f}` path the log statements use.
|
||||
fn expectRedacted(expected: []const u8, url: []const u8) !void {
|
||||
var buf: [512]u8 = undefined;
|
||||
const got = try std.fmt.bufPrint(&buf, "{f}", .{redactUrl(url)});
|
||||
try std.testing.expectEqualStrings(expected, got);
|
||||
}
|
||||
|
||||
test "redactUrl: hides every in-tree provider's credential parameter" {
|
||||
// Tiingo (`token`) - the key that actually leaked.
|
||||
try expectRedacted(
|
||||
"https://api.tiingo.com/tiingo/daily/SPY/prices?startDate=2026-08-07&endDate=2026-08-17&token=REDACTED",
|
||||
"https://api.tiingo.com/tiingo/daily/SPY/prices?startDate=2026-08-07&endDate=2026-08-17&token=0123456789abcdef0123456789abcdef01234567",
|
||||
);
|
||||
// FMP and TwelveData (`apikey`).
|
||||
try expectRedacted(
|
||||
"https://financialmodelingprep.com/stable/earnings?symbol=SMPL&apikey=REDACTED",
|
||||
"https://financialmodelingprep.com/stable/earnings?symbol=SMPL&apikey=cafebabecafebabecafebabecafebabe",
|
||||
);
|
||||
// Polygon (`apiKey`) - different case, and hand-rolled rather than
|
||||
// built by `buildUrl`.
|
||||
try expectRedacted(
|
||||
"https://api.polygon.io/v3/reference/dividends?ticker=SMPL&apiKey=REDACTED",
|
||||
"https://api.polygon.io/v3/reference/dividends?ticker=SMPL&apiKey=deadbeefdeadbeefdeadbeefdeadbeef",
|
||||
);
|
||||
}
|
||||
|
||||
test "redactUrl: keeps the diagnostically useful parts" {
|
||||
// The point of these log lines is "which request failed?". Symbol,
|
||||
// dates and interval must survive so they stay actionable.
|
||||
try expectRedacted(
|
||||
"https://api.example.com/v1/bars?symbol=SMPL&interval=1d&startDate=2026-08-07&adjusted=true&token=REDACTED",
|
||||
"https://api.example.com/v1/bars?symbol=SMPL&interval=1d&startDate=2026-08-07&adjusted=true&token=0123456789abcdef0123456789abcdef01234567",
|
||||
);
|
||||
// A comma-separated symbol list is long but is not a credential.
|
||||
try expectRedacted(
|
||||
"https://api.example.com/quotes?symbols=SMPLA,SMPLB,SMPLC,SMPLD,SMPLE",
|
||||
"https://api.example.com/quotes?symbols=SMPLA,SMPLB,SMPLC,SMPLD,SMPLE",
|
||||
);
|
||||
}
|
||||
|
||||
test "redactUrl: passes through URLs with no query string" {
|
||||
try expectRedacted("https://zfin.example.org/SPY/diagnostics", "https://zfin.example.org/SPY/diagnostics");
|
||||
try expectRedacted("https://zfin.example.org/SPY/candles?", "https://zfin.example.org/SPY/candles?");
|
||||
}
|
||||
|
||||
test "redactUrl: tolerates malformed query segments" {
|
||||
// A bare flag with no '=' must not be dropped or mangled.
|
||||
try expectRedacted(
|
||||
"https://api.example.com/x?flag&symbol=SMPL&token=REDACTED",
|
||||
"https://api.example.com/x?flag&symbol=SMPL&token=0123456789abcdef0123456789abcdef01234567",
|
||||
);
|
||||
// An empty credential value stays redacted rather than revealing
|
||||
// that the key was absent.
|
||||
try expectRedacted("https://api.example.com/x?token=REDACTED", "https://api.example.com/x?token=");
|
||||
}
|
||||
|
||||
test "redactUrl: heuristic backstops an unlisted parameter name" {
|
||||
// If a future provider authenticates with a name nobody added to
|
||||
// `credential_params`, an opaque 32-byte value still gets caught.
|
||||
try expectRedacted(
|
||||
"https://api.example.com/x?symbol=SMPL&sessionCredential=REDACTED",
|
||||
"https://api.example.com/x?symbol=SMPL&sessionCredential=cafebabecafebabecafebabecafebabe",
|
||||
);
|
||||
}
|
||||
|
||||
test "looksLikeCredential: separates opaque secrets from real query values" {
|
||||
// Secrets: 32-byte hex, 40-byte hex, base62-with-dash-and-underscore.
|
||||
try std.testing.expect(looksLikeCredential("cafebabecafebabecafebabecafebabe"));
|
||||
try std.testing.expect(looksLikeCredential("0123456789abcdef0123456789abcdef01234567"));
|
||||
try std.testing.expect(looksLikeCredential("ya29_A0ARrdaM-abcdefghijklmnop"));
|
||||
|
||||
// Everything these providers legitimately send is either short...
|
||||
try std.testing.expect(!looksLikeCredential("SMPL"));
|
||||
try std.testing.expect(!looksLikeCredential("2026-08-07"));
|
||||
try std.testing.expect(!looksLikeCredential("1d"));
|
||||
try std.testing.expect(!looksLikeCredential("true"));
|
||||
try std.testing.expect(!looksLikeCredential("0000320193"));
|
||||
// ...or long but punctuated in ways a key never is.
|
||||
try std.testing.expect(!looksLikeCredential("SMPLA,SMPLB,SMPLC,SMPLD,SMPLE"));
|
||||
try std.testing.expect(!looksLikeCredential("2026-08-07T00:00:00.000Z"));
|
||||
}
|
||||
|
||||
test "isCredentialParam: case-insensitive across provider spellings" {
|
||||
try std.testing.expect(isCredentialParam("token"));
|
||||
try std.testing.expect(isCredentialParam("apikey"));
|
||||
try std.testing.expect(isCredentialParam("apiKey"));
|
||||
try std.testing.expect(isCredentialParam("APIKEY"));
|
||||
try std.testing.expect(isCredentialParam("api_key"));
|
||||
try std.testing.expect(isCredentialParam("access_token"));
|
||||
|
||||
try std.testing.expect(!isCredentialParam("symbol"));
|
||||
try std.testing.expect(!isCredentialParam("ticker"));
|
||||
try std.testing.expect(!isCredentialParam("startDate"));
|
||||
}
|
||||
|
|
|
|||
624
src/service.zig
624
src/service.zig
|
|
@ -23,6 +23,7 @@ const Holding = @import("models/etf_profile.zig").Holding;
|
|||
const SectorWeight = @import("models/etf_profile.zig").SectorWeight;
|
||||
const Config = @import("Config.zig");
|
||||
const cache = @import("cache/store.zig");
|
||||
const freshness = @import("cache/freshness.zig");
|
||||
const srf = @import("srf");
|
||||
const analysis = @import("analytics/analysis.zig");
|
||||
const transaction_log = @import("models/transaction_log.zig");
|
||||
|
|
@ -133,6 +134,17 @@ pub fn isPermanentProviderFailure(err: anyerror) bool {
|
|||
return err == error.NotFound;
|
||||
}
|
||||
|
||||
/// Spread applied to `Ttl.tiingo_backoff` when a symbol is demoted off
|
||||
/// Tiingo by a 404.
|
||||
///
|
||||
/// Policy lives here rather than in `TtlSpec` (see its doc comment).
|
||||
/// 7% of 30 days is roughly +/-2 days, so a batch of symbols demoted in
|
||||
/// the same window re-probes across five distinct days instead of all
|
||||
/// on one. Sized against the cron cadence: with the refresh running
|
||||
/// twice a day, five days of spread keeps the re-probe burst to a
|
||||
/// handful of extra 404s per run.
|
||||
const tiingo_backoff_jitter_pct: u8 = 7;
|
||||
|
||||
/// Result of a CUSIP-to-ticker lookup (provider-agnostic).
|
||||
pub const CusipResult = OpenFigi.FigiResult;
|
||||
|
||||
|
|
@ -652,7 +664,7 @@ pub const DataService = struct {
|
|||
// (market-aware next post-close / NAV-availability time).
|
||||
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
|
||||
const kind = market.classify(symbol);
|
||||
s.cacheCandles(symbol, triple.candles, .tiingo, 0, expiryAfterFetch(now_s, kind, triple.candles));
|
||||
s.cacheCandles(symbol, triple.candles, .{ .provider = .tiingo }, expiryAfterFetch(now_s, kind, triple.candles));
|
||||
}
|
||||
// Dividends and splits use the supplement write path: Tiingo's
|
||||
// view merges into existing (typically Polygon-sourced) records
|
||||
|
|
@ -666,6 +678,112 @@ pub const DataService = struct {
|
|||
return triple;
|
||||
}
|
||||
|
||||
/// Fixed start date for any full-history candle fetch. See
|
||||
/// `populateAllFromTiingo`'s doc comment for the rationale.
|
||||
const full_history_start: Date = Date.fromYmd(2000, 1, 1);
|
||||
|
||||
/// Replace a symbol's entire candle series, restating `adj_close`
|
||||
/// from whichever provider will serve it.
|
||||
///
|
||||
/// This is the only way a cached series' adjustment basis ever
|
||||
/// advances - see `CandleMeta.adj_basis`. Incremental appends
|
||||
/// cannot restate the rows already on disk, so once a distribution
|
||||
/// goes ex behind them the whole series reads low until something
|
||||
/// rewrites it. That something is this function.
|
||||
///
|
||||
/// Tiingo first (it returns dividends and splits in the same
|
||||
/// response, and its `adj_close` is the one the analytics layer is
|
||||
/// written against), falling back to a full-range Yahoo fetch when
|
||||
/// Tiingo does not carry the symbol.
|
||||
///
|
||||
/// **Never writes a negative-cache entry.** Callers reach this
|
||||
/// holding a working series; `writeNegative` would overwrite
|
||||
/// `candles_daily.srf` with a marker and destroy that history. A
|
||||
/// restatement that cannot be completed is a "try again later", not
|
||||
/// a verdict about the symbol. Callers should treat any error as
|
||||
/// "keep the existing series and carry on".
|
||||
///
|
||||
/// Returns `error.NotFound` only when *every* provider says it does
|
||||
/// not carry the symbol - that is the one outcome a cold-start
|
||||
/// caller may legitimately turn into a negative-cache entry. Any
|
||||
/// other failure surfaces as `FetchFailed` / `TransientError` /
|
||||
/// `AuthError` so a network blip can never be mistaken for
|
||||
/// "this symbol does not exist".
|
||||
fn refetchFullHistory(
|
||||
self: *DataService,
|
||||
symbol: []const u8,
|
||||
today: Date,
|
||||
now_s: i64,
|
||||
prefer_yahoo: bool,
|
||||
) (DataError || error{NotFound})![]Candle {
|
||||
self.assertNetworkAllowed("getCandles refetchFullHistory");
|
||||
const kind = market.classify(symbol);
|
||||
var s = self.store();
|
||||
|
||||
// Tracks whether each provider affirmatively said "no such
|
||||
// symbol", as opposed to failing for some other reason. Only
|
||||
// unanimous 404s justify the caller writing a negative entry.
|
||||
var tiingo_not_found = false;
|
||||
|
||||
if (!prefer_yahoo) {
|
||||
if (self.populateAllFromTiingo(symbol)) |triple| {
|
||||
defer Dividend.freeSlice(self.allocator, triple.dividends);
|
||||
defer self.allocator.free(triple.splits);
|
||||
if (triple.candles.len > 0) return triple.candles;
|
||||
// An empty full-history response is not a restatement;
|
||||
// fall through rather than caching emptiness.
|
||||
self.allocator.free(triple.candles);
|
||||
log.warn("{s}: Tiingo full history returned no bars, trying Yahoo", .{symbol});
|
||||
} else |err| {
|
||||
// Transient failures must not silently degrade to a
|
||||
// second provider - the caller needs to know to retry.
|
||||
if (err == error.RateLimited or isTransientError(err)) return DataError.TransientError;
|
||||
if (err == error.Unauthorized) {
|
||||
log.err("{s}: Tiingo auth failed during restatement - check TIINGO_API_KEY", .{symbol});
|
||||
return DataError.AuthError;
|
||||
}
|
||||
tiingo_not_found = isPermanentProviderFailure(err);
|
||||
log.info("{s}: Tiingo cannot serve full history ({s}), trying Yahoo", .{ symbol, @errorName(err) });
|
||||
}
|
||||
}
|
||||
|
||||
// Yahoo fallback. `fetchCandles` takes an arbitrary range, so a
|
||||
// full-history request is just a wide one. Yahoo carries no
|
||||
// dividend/split endpoint here; those caches are Polygon-primary
|
||||
// and merged separately, so leaving them untouched is correct.
|
||||
if (self.getProvider(Yahoo)) |yh| {
|
||||
if (yh.fetchCandles(self.allocator, symbol, full_history_start, today)) |candles| {
|
||||
if (candles.len > 0) {
|
||||
s.cacheCandles(symbol, candles, .{
|
||||
.provider = .yahoo,
|
||||
// Preserve the Tiingo verdict this call just
|
||||
// learned, so the next pass doesn't re-probe.
|
||||
.tiingo_retry_after_s = if (tiingo_not_found)
|
||||
cache.computeExpires(
|
||||
now_s,
|
||||
.{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct },
|
||||
symbol,
|
||||
)
|
||||
else
|
||||
0,
|
||||
}, expiryAfterFetch(now_s, kind, candles));
|
||||
log.info("{s}: full history restated from Yahoo ({d} bars)", .{ symbol, candles.len });
|
||||
return candles;
|
||||
}
|
||||
self.allocator.free(candles);
|
||||
log.warn("{s}: Yahoo full history returned no bars", .{symbol});
|
||||
} else |err| {
|
||||
log.warn("{s}: Yahoo full history failed: {s}", .{ symbol, @errorName(err) });
|
||||
// Both providers affirmatively disclaim the symbol.
|
||||
if (tiingo_not_found and isPermanentProviderFailure(err)) return error.NotFound;
|
||||
}
|
||||
} else |_| {
|
||||
log.warn("{s}: Yahoo provider not available for full history", .{symbol});
|
||||
}
|
||||
|
||||
return DataError.FetchFailed;
|
||||
}
|
||||
|
||||
/// Invalidate cached data for a symbol so the next get* call forces a fresh fetch.
|
||||
pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void {
|
||||
var s = self.store();
|
||||
|
|
@ -678,6 +796,73 @@ pub const DataService = struct {
|
|||
|
||||
// ── Public data methods ──────────────────────────────────────
|
||||
|
||||
/// What a candle fetch learned about Tiingo's coverage of a symbol.
|
||||
///
|
||||
/// This is deliberately three-valued. The old code collapsed
|
||||
/// "Tiingo does not carry this symbol" and "Tiingo failed this
|
||||
/// request" into a single permanent demotion to Yahoo, which is
|
||||
/// how 22 of 32 cached symbols drifted off Tiingo. Only
|
||||
/// `.not_found` is a statement about the symbol; everything else
|
||||
/// is a statement about one HTTP call and must not be remembered.
|
||||
pub const TiingoCoverage = enum {
|
||||
/// Tiingo served the request - it definitely covers this symbol.
|
||||
covered,
|
||||
/// Tiingo returned a genuine 404 - it does not carry this symbol.
|
||||
not_found,
|
||||
/// Tiingo was not consulted, or failed for a reason that says
|
||||
/// nothing about coverage (active backoff, no API key, 400,
|
||||
/// 402, malformed body). Nothing new learned; leave any
|
||||
/// existing backoff exactly as it was.
|
||||
unknown,
|
||||
};
|
||||
|
||||
/// Fold a fetch's outcome into a symbol's candle metadata.
|
||||
///
|
||||
/// Called on any *successful* candle fetch, so `fail_count` resets
|
||||
/// to 0. The interesting part is `tiingo_retry_after_s`:
|
||||
///
|
||||
/// - `.covered` -> clear the backoff. Tiingo just served this
|
||||
/// symbol, so any prior 404 is stale news. This
|
||||
/// is what walks a previously-demoted symbol
|
||||
/// back onto Tiingo.
|
||||
/// - `.not_found` -> arm the backoff at `Ttl.tiingo_backoff` with
|
||||
/// per-symbol jitter, so a batch demoted in the
|
||||
/// same window does not all re-probe on one day.
|
||||
/// - `.unknown` -> leave it untouched. Either Tiingo was never
|
||||
/// asked, or it failed for a reason that says
|
||||
/// nothing about coverage.
|
||||
fn applyTiingoCoverage(
|
||||
meta: cache.Store.CandleMeta,
|
||||
symbol: []const u8,
|
||||
now_s: i64,
|
||||
provider: cache.Store.CandleProvider,
|
||||
coverage: TiingoCoverage,
|
||||
) cache.Store.CandleMeta {
|
||||
var next = meta;
|
||||
next.provider = provider;
|
||||
next.fail_count = 0;
|
||||
|
||||
switch (coverage) {
|
||||
.covered => {
|
||||
if (meta.tiingo_retry_after_s != 0) {
|
||||
log.info("{s}: provider converted {t} -> tiingo, clearing Tiingo backoff", .{ symbol, meta.provider });
|
||||
}
|
||||
next.tiingo_retry_after_s = 0;
|
||||
},
|
||||
.not_found => {
|
||||
next.tiingo_retry_after_s = cache.computeExpires(
|
||||
now_s,
|
||||
.{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct },
|
||||
symbol,
|
||||
);
|
||||
log.info("{s}: Tiingo NotFound, backing off until {f}", .{ symbol, Date.fromEpoch(next.tiingo_retry_after_s) });
|
||||
},
|
||||
.unknown => {},
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
/// Fetch candles from providers with error classification.
|
||||
///
|
||||
/// Error handling:
|
||||
|
|
@ -685,32 +870,42 @@ pub const DataService = struct {
|
|||
/// - NotFound/ParseError/InvalidResponse from Tiingo -> try Yahoo (symbol-level issue)
|
||||
/// - Unauthorized -> TransientError (config problem, stop refresh)
|
||||
///
|
||||
/// The `preferred` param controls incremental fetch consistency: use the same
|
||||
/// provider that sourced the existing cache data.
|
||||
/// `prefer_yahoo` skips the Tiingo attempt entirely. Callers derive
|
||||
/// it from `CandleMeta.tiingo_retry_after_s` - i.e. "Tiingo told us
|
||||
/// 404 recently, don't waste the call". It is NOT derived from
|
||||
/// which provider sourced the cache; see the `provider` field's
|
||||
/// doc comment for why that distinction matters.
|
||||
fn fetchCandlesFromProviders(
|
||||
self: *DataService,
|
||||
symbol: []const u8,
|
||||
from: Date,
|
||||
to: Date,
|
||||
preferred: cache.Store.CandleProvider,
|
||||
) (DataError || error{NotFound})!struct { candles: []Candle, provider: cache.Store.CandleProvider } {
|
||||
// If preferred is Yahoo (degraded symbol), try Yahoo first
|
||||
if (preferred == .yahoo) {
|
||||
prefer_yahoo: bool,
|
||||
) (DataError || error{NotFound})!struct {
|
||||
candles: []Candle,
|
||||
provider: cache.Store.CandleProvider,
|
||||
tiingo_coverage: TiingoCoverage,
|
||||
} {
|
||||
// Under an active Tiingo backoff, go straight to Yahoo.
|
||||
if (prefer_yahoo) {
|
||||
if (self.getProvider(Yahoo)) |yh| {
|
||||
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Yahoo (preferred)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .yahoo };
|
||||
log.debug("{s}: candles from Yahoo (Tiingo backoff active)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = .unknown };
|
||||
} else |err| {
|
||||
log.warn("{s}: Yahoo (preferred) failed: {s}", .{ symbol, @errorName(err) });
|
||||
log.warn("{s}: Yahoo (Tiingo backoff active) failed: {s}", .{ symbol, @errorName(err) });
|
||||
}
|
||||
} else |_| {}
|
||||
}
|
||||
|
||||
// Primary: Tiingo
|
||||
// Primary: Tiingo. `coverage` accumulates what this attempt
|
||||
// taught us, so the Yahoo fallback below can report it back
|
||||
// to the caller without re-deriving it from the error.
|
||||
var coverage: TiingoCoverage = .unknown;
|
||||
if (self.getProvider(Tiingo)) |tg| {
|
||||
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Tiingo", .{symbol});
|
||||
return .{ .candles = candles, .provider = .tiingo };
|
||||
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
|
||||
} else |err| {
|
||||
log.warn("{s}: Tiingo failed: {s}", .{ symbol, @errorName(err) });
|
||||
|
||||
|
|
@ -725,7 +920,7 @@ pub const DataService = struct {
|
|||
self.rateLimitBackoff();
|
||||
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Tiingo (after rate limit backoff)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .tiingo };
|
||||
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
|
||||
} else |retry_err| {
|
||||
log.warn("{s}: Tiingo retry after backoff failed: {s}", .{ symbol, @errorName(retry_err) });
|
||||
if (retry_err == error.RateLimited) {
|
||||
|
|
@ -733,7 +928,7 @@ pub const DataService = struct {
|
|||
self.rateLimitBackoff();
|
||||
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Tiingo (after second backoff)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .tiingo };
|
||||
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
|
||||
} else |_| {}
|
||||
}
|
||||
// Exhausted rate limit retries - treat as transient
|
||||
|
|
@ -746,19 +941,31 @@ pub const DataService = struct {
|
|||
return DataError.TransientError;
|
||||
}
|
||||
|
||||
// NotFound, ParseError, InvalidResponse - symbol-level issue, try Yahoo
|
||||
log.info("{s}: Tiingo does not have this symbol, trying Yahoo", .{symbol});
|
||||
// NotFound, ParseError, InvalidResponse - fall back to
|
||||
// Yahoo for this call. Only a genuine 404 is a
|
||||
// statement about Tiingo's coverage; a 400 / 402 /
|
||||
// malformed body says something about this request and
|
||||
// must not earn a remembered demotion. Mirrors the rule
|
||||
// `isPermanentProviderFailure` already applies to the
|
||||
// negative cache.
|
||||
if (isPermanentProviderFailure(err)) {
|
||||
coverage = .not_found;
|
||||
log.info("{s}: Tiingo does not carry this symbol, trying Yahoo", .{symbol});
|
||||
} else {
|
||||
log.info("{s}: Tiingo request failed ({s}) - not a coverage verdict, trying Yahoo for this call only", .{ symbol, @errorName(err) });
|
||||
}
|
||||
}
|
||||
} else |_| {
|
||||
log.warn("{s}: Tiingo provider not available (no API key?)", .{symbol});
|
||||
}
|
||||
|
||||
// Fallback: Yahoo (symbol not on Tiingo)
|
||||
if (preferred != .yahoo) {
|
||||
// Fallback: Yahoo. Skipped when we already tried Yahoo first
|
||||
// (active backoff) and it failed - no point asking twice.
|
||||
if (!prefer_yahoo) {
|
||||
if (self.getProvider(Yahoo)) |yh| {
|
||||
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.info("{s}: candles from Yahoo (Tiingo fallback)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .yahoo };
|
||||
return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = coverage };
|
||||
} else |err| {
|
||||
log.warn("{s}: Yahoo fallback also failed: {s}", .{ symbol, @errorName(err) });
|
||||
}
|
||||
|
|
@ -932,7 +1139,17 @@ pub const DataService = struct {
|
|||
// (Force-refresh skips server sync too: the user explicitly
|
||||
// asked for fresh provider data.)
|
||||
if (!opts.force_refresh and self.syncCandlesFromServer(symbol)) {
|
||||
if (s.isCandleMetaFresh(symbol)) {
|
||||
// Re-read meta: the sync wrote the server's bytes
|
||||
// verbatim, so its view of both freshness AND
|
||||
// adjustment basis is now ours. `serverBarRegression`
|
||||
// only guards `last_date` going backwards - it cannot
|
||||
// see a same-dated file whose historical adj_close is
|
||||
// stale, so the basis has to be re-checked here.
|
||||
const synced = if (s.readCandleMeta(symbol)) |sm| sm.meta else m;
|
||||
const synced_action = freshness.newestCorporateAction(self.allocator, &s, symbol);
|
||||
if (s.isCandleMetaFresh(symbol) and
|
||||
!freshness.adjustmentBasisStale(synced.adj_basis, synced.last_date, synced_action))
|
||||
{
|
||||
log.debug("{s}: candles synced from server and fresh", .{symbol});
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
|
|
@ -947,6 +1164,36 @@ pub const DataService = struct {
|
|||
// this stale path (next post-close / NAV-availability time).
|
||||
const expires = market.nextCandleExpiry(now_s, kind);
|
||||
|
||||
// A corporate action has gone ex since the basis that
|
||||
// produced this series, so the cached `adj_close` values
|
||||
// behind it were never marked down. Appending cannot fix
|
||||
// that - only replacing the file can. Do this BEFORE the
|
||||
// incremental fetch so a symbol that needs both a top-up
|
||||
// and a restatement costs one full fetch, not an append
|
||||
// followed by a second pass.
|
||||
//
|
||||
// Deliberately not checked on the fresh-cache path above:
|
||||
// the candle TTL lapses at least once per trading day, so
|
||||
// detection is at most a day behind, and paying a
|
||||
// dividend/split cache read on every getCandles call would
|
||||
// tax the hot portfolio-pricing path for nothing.
|
||||
if (freshness.adjustmentBasisStale(
|
||||
m.adj_basis,
|
||||
m.last_date,
|
||||
freshness.newestCorporateAction(self.allocator, &s, symbol),
|
||||
)) {
|
||||
log.info("{s}: restating full history (adj_basis {f} predates a corporate action)", .{ symbol, m.adj_basis });
|
||||
if (self.refetchFullHistory(symbol, today, now_s, now_s < m.tiingo_retry_after_s)) |candles| {
|
||||
return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
} else |err| {
|
||||
// Restatement is best-effort. The existing series
|
||||
// is untouched and still usable, just understated
|
||||
// by the missed adjustment - fall through to the
|
||||
// normal top-up and retry on the next stale pass.
|
||||
log.warn("{s}: full-history restatement failed ({s}); adjustment basis remains stale, falling back to incremental append", .{ symbol, @errorName(err) });
|
||||
}
|
||||
}
|
||||
|
||||
// Only skip the fetch when we already hold the latest
|
||||
// *available* bar (weekend/holiday/pre-close gap, or
|
||||
// caught up): just bump the TTL. Gating on the
|
||||
|
|
@ -959,18 +1206,20 @@ pub const DataService = struct {
|
|||
// boundary while the lag check reported it lagging (the
|
||||
// Friday-17:00 deadlock that exited 75 every retry).
|
||||
if (!market.shouldRefresh(now_s, kind, m.last_date)) {
|
||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires);
|
||||
s.updateCandleMeta(symbol, m, expires);
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
} else {
|
||||
// Incremental fetch from day after last cached candle
|
||||
self.assertNetworkAllowed("getCandles incremental fetchCandlesFromProviders");
|
||||
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch |err| {
|
||||
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, now_s < m.tiingo_retry_after_s) catch |err| {
|
||||
if (err == DataError.TransientError) {
|
||||
// Increment fail_count for this symbol
|
||||
const new_fail_count = m.fail_count +| 1; // saturating add
|
||||
log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
|
||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s);
|
||||
var degraded = m;
|
||||
degraded.fail_count = new_fail_count;
|
||||
s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s);
|
||||
|
||||
// If degraded (fail_count >= 3), return stale data rather than failing
|
||||
if (new_fail_count >= 3) {
|
||||
|
|
@ -987,6 +1236,10 @@ pub const DataService = struct {
|
|||
};
|
||||
const new_candles = result.candles;
|
||||
|
||||
// Fold what this fetch learned about Tiingo coverage
|
||||
// into the metadata we're about to write.
|
||||
const next_meta = applyTiingoCoverage(m, symbol, now_s, result.provider, result.tiingo_coverage);
|
||||
|
||||
if (new_candles.len == 0) {
|
||||
// No new candles. Either a genuine non-trading-day
|
||||
// gap (weekend/holiday), the provider hasn't posted
|
||||
|
|
@ -999,7 +1252,7 @@ pub const DataService = struct {
|
|||
// an un-modeled closure stops thrashing and waits
|
||||
// for the next real session.
|
||||
self.allocator.free(new_candles);
|
||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date));
|
||||
s.updateCandleMeta(symbol, next_meta, market.staleCandleExpiry(now_s, kind, m.last_date));
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
} else {
|
||||
|
|
@ -1007,7 +1260,7 @@ pub const DataService = struct {
|
|||
// TTL via `expiryAfterFetch`, NOT the precomputed
|
||||
// next-boundary `expires`: getting a bar back does not
|
||||
// mean getting the RIGHT bar back.
|
||||
s.appendCandles(symbol, new_candles, result.provider, 0, expiryAfterFetch(now_s, kind, new_candles));
|
||||
s.appendCandles(symbol, new_candles, next_meta, expiryAfterFetch(now_s, kind, new_candles));
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
|
||||
self.allocator.free(new_candles);
|
||||
return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
|
|
@ -1034,45 +1287,50 @@ pub const DataService = struct {
|
|||
log.debug("{s}: candles synced from server but stale, falling through to full fetch", .{symbol});
|
||||
}
|
||||
|
||||
// No usable cache - full fetch via the orchestrated Tiingo
|
||||
// helper, which writes candles + dividends + splits caches in
|
||||
// one shot from a single HTTP response. The fixed start date
|
||||
// (see `populateAllFromTiingo`) is 2000-01-01, deep enough to
|
||||
// cover a 10Y trailing-return window even when `--as-of`
|
||||
// back-dates the reference into 2014-era imported portfolio
|
||||
// history, plus a buffer for older corporate actions like
|
||||
// SPYM's 2017-10-16 split.
|
||||
// No usable cache - full fetch. Tiingo first (it returns
|
||||
// candles + dividends + splits from one response), falling back
|
||||
// to a full-range Yahoo fetch when Tiingo does not carry the
|
||||
// symbol. The fixed start date (see `populateAllFromTiingo`) is
|
||||
// 2000-01-01, deep enough to cover a 10Y trailing-return window
|
||||
// even when `--as-of` back-dates the reference into 2014-era
|
||||
// imported portfolio history, plus a buffer for older corporate
|
||||
// actions like SPYM's 2017-10-16 split.
|
||||
//
|
||||
// The Yahoo fallback matters here beyond convenience: this
|
||||
// branch used to be Tiingo-only, so a symbol Tiingo does not
|
||||
// carry could not be cold-started at all - it 404'd, wrote a
|
||||
// negative-cache marker *over* candles_daily.srf, and stayed
|
||||
// unavailable until `cache clear`. Any symbol that reached this
|
||||
// branch with a populated cache would have had its history
|
||||
// destroyed.
|
||||
log.debug("{s}: fetching full candle history from provider", .{symbol});
|
||||
self.assertNetworkAllowed("getCandles full populateAllFromTiingo");
|
||||
|
||||
const triple = self.populateAllFromTiingo(symbol) catch |err| {
|
||||
if (err == error.RateLimited or err == error.ServerError or err == error.RequestFailed) {
|
||||
const prior_backoff: i64 = if (meta_result) |mr| mr.meta.tiingo_retry_after_s else 0;
|
||||
const candles = self.refetchFullHistory(symbol, today, now_s, now_s < prior_backoff) catch |err| {
|
||||
if (err == DataError.TransientError) {
|
||||
// Transient: increment fail_count on existing meta so
|
||||
// we know to back off if this keeps happening.
|
||||
if (meta_result) |mr| {
|
||||
const new_fail_count = mr.meta.fail_count +| 1;
|
||||
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s);
|
||||
var degraded = mr.meta;
|
||||
degraded.fail_count = mr.meta.fail_count +| 1;
|
||||
s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s);
|
||||
}
|
||||
return DataError.TransientError;
|
||||
}
|
||||
// Only a genuine NotFound means "this symbol has no candle
|
||||
// data on Tiingo" (the sole historical-candle provider since
|
||||
// the 2026-05 audit) and earns a sticky negative-cache entry.
|
||||
// Unauthorized / InvalidResponse / PaymentRequired and the
|
||||
// like are not permanent facts about the symbol (transient
|
||||
// auth misconfig, a malformed response) - fail this call but
|
||||
// stay retryable, matching fetchCached's policy. This matters
|
||||
// now that the candle negative cache is actually honored: a
|
||||
// bad negative would otherwise stick until --refresh.
|
||||
if (isPermanentProviderFailure(err)) s.writeNegative(symbol, .candles_daily);
|
||||
// Only a unanimous NotFound - every provider affirmatively
|
||||
// disclaims the symbol - earns a sticky negative-cache
|
||||
// entry. Auth trouble, a malformed response, or a Yahoo
|
||||
// network blip are not permanent facts about the symbol, so
|
||||
// they fail this call but stay retryable. This matters
|
||||
// because the negative marker replaces the candle file: a
|
||||
// bad negative both suppresses retries until `--refresh`
|
||||
// and throws away whatever history was cached.
|
||||
if (err == error.NotFound) s.writeNegative(symbol, .candles_daily);
|
||||
if (err == DataError.AuthError) return DataError.AuthError;
|
||||
return DataError.FetchFailed;
|
||||
};
|
||||
// populateAllFromTiingo writes all three caches itself; we
|
||||
// free the slices we don't return.
|
||||
defer Dividend.freeSlice(self.allocator, triple.dividends);
|
||||
defer self.allocator.free(triple.splits);
|
||||
|
||||
return .{ .data = triple.candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
return .{ .data = candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
}
|
||||
|
||||
/// Fetch dividend history for a symbol.
|
||||
|
|
@ -1983,9 +2241,18 @@ pub const DataService = struct {
|
|||
/// See `tmp/multi-ticker-audit.md` for the cross-validation evidence.
|
||||
pub fn getTrailingReturns(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!struct {
|
||||
asof_price: performance.TrailingReturns,
|
||||
asof_total: ?performance.TrailingReturns,
|
||||
/// Total return as of the newest bar. Non-optional: it is always
|
||||
/// computable once candles exist, because `performance.totalReturns`
|
||||
/// degrades to the adj_close series internally when no dividend
|
||||
/// record is available. It was `?TrailingReturns`, and the
|
||||
/// optionality invited consumers to write
|
||||
/// `asof_total orelse asof_price` - silently publishing a
|
||||
/// price-only return labelled as total return, understated by the
|
||||
/// full dividend yield.
|
||||
asof_total: performance.TrailingReturns,
|
||||
me_price: performance.TrailingReturns,
|
||||
me_total: ?performance.TrailingReturns,
|
||||
/// Month-end total return. Non-optional for the same reason.
|
||||
me_total: performance.TrailingReturns,
|
||||
candles: []Candle,
|
||||
dividends: ?[]Dividend,
|
||||
source: Source,
|
||||
|
|
@ -2014,34 +2281,17 @@ pub const DataService = struct {
|
|||
// Month-end (end = last business day of prior month)
|
||||
const me_price = performance.trailingReturnsPriceOnlyMonthEnd(c, splits, today);
|
||||
|
||||
// Total return: dividend-reinvested when dividends are
|
||||
// available; otherwise fall back to adj_close-based total
|
||||
// return (which captures dividends for providers like Tiingo
|
||||
// that bake dividends into adj_close).
|
||||
// Total return. `performance.totalReturns` owns the
|
||||
// dividend-reinvestment vs adj_close merge, including the
|
||||
// no-dividend-record fallback, so there is one code path here
|
||||
// rather than a branch per availability case.
|
||||
var divs: ?[]Dividend = null;
|
||||
var asof_total: ?performance.TrailingReturns = null;
|
||||
var me_total: ?performance.TrailingReturns = null;
|
||||
|
||||
// adj_close-based total return is the fallback. With Tiingo
|
||||
// (the default provider) adj_close is already dividend-
|
||||
// adjusted, so this gives a reasonable total-return estimate
|
||||
// even when explicit dividend records are missing.
|
||||
const asof_adj = performance.trailingReturns(c);
|
||||
const me_adj = performance.trailingReturnsMonthEnd(c, today);
|
||||
|
||||
if (self.getDividends(symbol, opts)) |div_result| {
|
||||
divs = div_result.data;
|
||||
const asof_div = performance.trailingReturnsWithDividends(c, div_result.data);
|
||||
const me_div = performance.trailingReturnsMonthEndWithDividends(c, div_result.data, today);
|
||||
asof_total = performance.withDividendFallback(asof_div, asof_adj);
|
||||
me_total = performance.withDividendFallback(me_div, me_adj);
|
||||
} else |_| {
|
||||
// No dividend data: still surface the adj_close-based
|
||||
// total return rather than null, since Tiingo's
|
||||
// adj_close already includes dividend adjustment.
|
||||
asof_total = asof_adj;
|
||||
me_total = me_adj;
|
||||
}
|
||||
} else |_| {}
|
||||
|
||||
const asof_total = performance.totalReturns(c, divs, today);
|
||||
const me_total = performance.totalReturnsMonthEnd(c, divs, today);
|
||||
|
||||
return .{
|
||||
.asof_price = asof_price,
|
||||
|
|
@ -2133,6 +2383,18 @@ pub const DataService = struct {
|
|||
return .{ .data = result.data, .source = .cached, .timestamp = result.timestamp, .allocator = allocator };
|
||||
}
|
||||
|
||||
/// Cache-only split read, mirroring `getCachedDividends`.
|
||||
///
|
||||
/// Splits travel with dividends wherever raw `close` is being
|
||||
/// adjusted: `close` is not split-adjusted, so a total-return series
|
||||
/// built from dividends alone reads a 2:1 split as a -50% month.
|
||||
/// Callers that fetch one should fetch both.
|
||||
pub fn getCachedSplits(self: *DataService, allocator: std.mem.Allocator, symbol: []const u8) ?FetchResult(Split) {
|
||||
var s = self.store();
|
||||
const result = s.read(allocator, Split, symbol, null, .any) orelse return null;
|
||||
return .{ .data = result.data, .source = .cached, .timestamp = result.timestamp, .allocator = allocator };
|
||||
}
|
||||
|
||||
// ── Portfolio price loading ──────────────────────────────────
|
||||
|
||||
/// Status emitted for each symbol during price loading.
|
||||
|
|
@ -3791,7 +4053,7 @@ test "getCandles offline mode returns cached data without network" {
|
|||
.{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||
.{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 },
|
||||
};
|
||||
store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
store.cacheCandles("TEST", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Set the test guard: any network call would panic. We expect
|
||||
// the offline-mode path NOT to touch the network.
|
||||
|
|
@ -3892,6 +4154,38 @@ test "loadAllDividends: honors skip_network for every symbol, and one miss does
|
|||
try std.testing.expect(svc.getCachedDividends(allocator, "TSTA") == null);
|
||||
}
|
||||
|
||||
test "getCachedSplits: reads cache only and reports absence" {
|
||||
// Splits travel with dividends wherever raw `close` is adjusted, so
|
||||
// this mirrors `getCachedDividends` exactly - including that an
|
||||
// absent symbol returns null rather than an empty slice, which is a
|
||||
// different fact (nothing cached vs. cached-and-never-split).
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
const config = Config{ .cache_dir = dir_path };
|
||||
var svc = DataService.init(io, allocator, config);
|
||||
defer svc.deinit();
|
||||
|
||||
var splits = [_]Split{
|
||||
.{ .date = Date.fromYmd(2024, 6, 10), .numerator = 10, .denominator = 1 },
|
||||
};
|
||||
var store = svc.store();
|
||||
store.write(Split, "TSTB", splits[0..], cache.DataType.splits.ttl());
|
||||
|
||||
svc.panic_on_network_attempt = true;
|
||||
|
||||
const b = svc.getCachedSplits(allocator, "TSTB") orelse return error.TestUnexpectedResult;
|
||||
defer b.deinit();
|
||||
try std.testing.expectEqual(@as(usize, 1), b.data.len);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 10.0), b.data[0].ratio(), 1e-9);
|
||||
|
||||
try std.testing.expect(svc.getCachedSplits(allocator, "TSTA") == null);
|
||||
}
|
||||
|
||||
test "loadAllDividends: empty symbol list is a no-op" {
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
|
|
@ -3946,7 +4240,7 @@ test "loadAllPrices offline mode skips network and returns cached" {
|
|||
var fresh_candles = [_]Candle{
|
||||
.{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||
};
|
||||
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
store.cacheCandles("FRESH", fresh_candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Symbol with no cache at all.
|
||||
// (no setup needed - just passes a symbol that doesn't exist)
|
||||
|
|
@ -3972,6 +4266,168 @@ test "loadAllPrices offline mode skips network and returns cached" {
|
|||
try std.testing.expectEqual(@as(usize, 1), result.failed_count);
|
||||
}
|
||||
|
||||
// ── adjustment-basis restatement ─────────────────────────────
|
||||
|
||||
test "getCandles offline never escalates a stale adjustment basis" {
|
||||
// Restatement does network I/O, so `skip_network` must return the
|
||||
// cached series untouched rather than escalating. Pinned with
|
||||
// `panic_on_network_attempt`, which fires inside
|
||||
// `refetchFullHistory`'s `assertNetworkAllowed` if this regresses.
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
const config = Config{ .cache_dir = dir_path };
|
||||
var svc = DataService.init(io, allocator, config);
|
||||
defer svc.deinit();
|
||||
|
||||
var store = svc.store();
|
||||
var candles = [_]Candle{
|
||||
.{ .date = Date.fromYmd(2026, 8, 12), .open = 10, .high = 10, .low = 10, .close = 10, .adj_close = 10, .volume = 1 },
|
||||
.{ .date = Date.fromYmd(2026, 8, 13), .open = 11, .high = 11, .low = 11, .close = 11, .adj_close = 11, .volume = 1 },
|
||||
};
|
||||
store.cacheCandles("SMPL", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Roll the basis back behind a cached ex-date so the series is
|
||||
// unambiguously due for restatement.
|
||||
var divs = [_]Dividend{.{ .ex_date = Date.fromYmd(2026, 7, 1), .amount = 1.0 }};
|
||||
store.write(Dividend, "SMPL", divs[0..], .{ .seconds = cache.Ttl.dividends });
|
||||
var meta = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta;
|
||||
meta.adj_basis = Date.fromYmd(2026, 6, 1);
|
||||
store.updateCandleMeta("SMPL", meta, 1); // expiry in the past => stale
|
||||
try std.testing.expect(freshness.adjustmentBasisStale(
|
||||
meta.adj_basis,
|
||||
meta.last_date,
|
||||
freshness.newestCorporateAction(allocator, &store, "SMPL"),
|
||||
));
|
||||
|
||||
svc.panic_on_network_attempt = true;
|
||||
const result = try svc.getCandles("SMPL", .{ .skip_network = true });
|
||||
defer result.deinit();
|
||||
|
||||
// Served from cache, series intact, basis untouched.
|
||||
try std.testing.expectEqual(@as(usize, 2), result.data.len);
|
||||
const after = (store.readCandleMeta("SMPL") orelse return error.NoCache).meta;
|
||||
try std.testing.expect(after.adj_basis.eql(Date.fromYmd(2026, 6, 1)));
|
||||
// And critically, no negative-cache marker was written over the
|
||||
// candle file.
|
||||
try std.testing.expect(!store.isNegative("SMPL", .candles_daily));
|
||||
}
|
||||
|
||||
// ── Tiingo coverage bookkeeping ──────────────────────────────//
|
||||
// Regression suite for the one-way drift to Yahoo. The old code
|
||||
// routed on `CandleMeta.provider == .yahoo`, so any non-transient
|
||||
// Tiingo failure latched permanently: Yahoo was tried first,
|
||||
// succeeded, rewrote `provider = .yahoo`, and Tiingo was never asked
|
||||
// again. 22 of 32 cached symbols had drifted off Tiingo that way.
|
||||
|
||||
test "applyTiingoCoverage: .covered clears an armed Tiingo backoff" {
|
||||
const armed = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .yahoo,
|
||||
.fail_count = 2,
|
||||
.tiingo_retry_after_s = 1_790_000_000,
|
||||
};
|
||||
const next = DataService.applyTiingoCoverage(armed, "SMPL", 1_787_000_000, .tiingo, .covered);
|
||||
|
||||
// Tiingo just served the symbol, so the prior 404 is stale news.
|
||||
try std.testing.expectEqual(@as(i64, 0), next.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(cache.Store.CandleProvider.tiingo, next.provider);
|
||||
// A successful fetch also resets the transient-failure counter.
|
||||
try std.testing.expectEqual(@as(u8, 0), next.fail_count);
|
||||
}
|
||||
|
||||
test "applyTiingoCoverage: .not_found arms a jittered backoff ~30 days out" {
|
||||
const now_s: i64 = 1_787_000_000;
|
||||
const meta = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
const next = DataService.applyTiingoCoverage(meta, "SMPL", now_s, .yahoo, .not_found);
|
||||
|
||||
// Yahoo answered, so provenance records Yahoo...
|
||||
try std.testing.expectEqual(cache.Store.CandleProvider.yahoo, next.provider);
|
||||
// ...and the backoff lands 30 days out, +/- the jitter window.
|
||||
const base = now_s + cache.Ttl.tiingo_backoff;
|
||||
const max_offset = @divFloor(cache.Ttl.tiingo_backoff * @as(i64, tiingo_backoff_jitter_pct), 100);
|
||||
try std.testing.expect(next.tiingo_retry_after_s >= base - max_offset);
|
||||
try std.testing.expect(next.tiingo_retry_after_s <= base + max_offset);
|
||||
// The spread must be meaningful but bounded: roughly +/-2 days.
|
||||
try std.testing.expect(max_offset >= 2 * std.time.s_per_day);
|
||||
try std.testing.expect(max_offset <= 3 * std.time.s_per_day);
|
||||
}
|
||||
|
||||
test "applyTiingoCoverage: .unknown leaves the backoff untouched" {
|
||||
// A 400 / 402 / malformed body says nothing about coverage. It
|
||||
// must neither arm a backoff (that's the drift bug) nor clear an
|
||||
// existing one (that would defeat the 404 we already recorded).
|
||||
const now_s: i64 = 1_787_000_000;
|
||||
|
||||
const unarmed = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
const a = DataService.applyTiingoCoverage(unarmed, "SMPL", now_s, .yahoo, .unknown);
|
||||
try std.testing.expectEqual(@as(i64, 0), a.tiingo_retry_after_s);
|
||||
|
||||
var armed = unarmed;
|
||||
armed.tiingo_retry_after_s = 1_790_000_000;
|
||||
const b = DataService.applyTiingoCoverage(armed, "SMPL", now_s, .yahoo, .unknown);
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), b.tiingo_retry_after_s);
|
||||
}
|
||||
|
||||
test "applyTiingoCoverage: backoff jitter is deterministic per symbol" {
|
||||
// Deterministic-by-symbol (Wyhash via computeExpires), NOT random:
|
||||
// repeated writes for the same symbol must not drift the deadline,
|
||||
// and distinct symbols must land on different days so a batch
|
||||
// demoted in one window does not all re-probe together.
|
||||
const now_s: i64 = 1_787_000_000;
|
||||
const meta = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
|
||||
const a1 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found);
|
||||
const a2 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found);
|
||||
try std.testing.expectEqual(a1.tiingo_retry_after_s, a2.tiingo_retry_after_s);
|
||||
|
||||
// Spread check across a handful of symbols: they must not all
|
||||
// collapse onto the same instant.
|
||||
const syms = [_][]const u8{ "SMPLA", "SMPLB", "SMPLC", "SMPLD", "SMPLE", "SMPLF" };
|
||||
var seen: [syms.len]i64 = undefined;
|
||||
for (syms, 0..) |sym, i| {
|
||||
seen[i] = DataService.applyTiingoCoverage(meta, sym, now_s, .yahoo, .not_found).tiingo_retry_after_s;
|
||||
}
|
||||
var distinct: usize = 0;
|
||||
for (seen, 0..) |v, i| {
|
||||
var is_new = true;
|
||||
for (seen[0..i]) |prev| {
|
||||
if (prev == v) is_new = false;
|
||||
}
|
||||
if (is_new) distinct += 1;
|
||||
}
|
||||
try std.testing.expect(distinct > 1);
|
||||
}
|
||||
|
||||
test "isPermanentProviderFailure gates which Tiingo errors are remembered" {
|
||||
// The rule that Commit 1 reuses for the Yahoo-demotion decision:
|
||||
// only a genuine 404 is a statement about the symbol. Everything
|
||||
// else describes one HTTP call.
|
||||
try std.testing.expect(isPermanentProviderFailure(error.NotFound));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.InvalidResponse));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.PaymentRequired));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.ParseError));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.RateLimited));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.Unauthorized));
|
||||
}
|
||||
|
||||
test "loadAllPrices force_refresh tops up without wiping the candle cache" {
|
||||
// Regression: force_refresh must mean "ignore TTL + incremental
|
||||
// top-up", NOT "delete the cache and re-download from scratch".
|
||||
|
|
@ -3998,7 +4454,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" {
|
|||
var candles = [_]Candle{
|
||||
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||
};
|
||||
store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
store.cacheCandles("HELD", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Any provider/network attempt now panics. If force_refresh wiped
|
||||
// the cache (old behavior), getCandles would fall through to a full
|
||||
|
|
@ -5114,7 +5570,7 @@ test "serverBarRegression: blocks a regression and reports both dates" {
|
|||
|
||||
var s = cache.Store.init(io, allocator, dir_path);
|
||||
// Local copy holds Monday's bar.
|
||||
s.updateCandleMeta("AAPL", 100.0, Date.fromYmd(2026, 8, 10), .tiingo, 0, 9999999999);
|
||||
s.updateCandleMeta("AAPL", .{ .last_close = 100.0, .last_date = Date.fromYmd(2026, 8, 10), .provider = .tiingo }, 9999999999);
|
||||
|
||||
const older = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-07,provider::tiingo\n";
|
||||
const same = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-10,provider::tiingo\n";
|
||||
|
|
|
|||
|
|
@ -1031,12 +1031,36 @@ fn loadData(state: *State, app: *App) void {
|
|||
};
|
||||
|
||||
if (state.view) |*v| v.deinit(app.allocator);
|
||||
|
||||
// Splits for the totals row's total-return index. Read straight
|
||||
// from the cache rather than through a worker: the dividends
|
||||
// worker above has already warmed the same Tiingo responses that
|
||||
// carry splits, so this is a local file read. The slices only need
|
||||
// to outlive `buildReview`, which reduces them to scalars.
|
||||
var split_map = std.StringHashMap([]const zfin.Split).init(app.allocator);
|
||||
defer {
|
||||
var it = split_map.iterator();
|
||||
while (it.next()) |entry| {
|
||||
app.allocator.free(@constCast(entry.value_ptr.*));
|
||||
}
|
||||
split_map.deinit();
|
||||
}
|
||||
for (summary_ptr.allocations) |a| {
|
||||
if (app.svc.getCachedSplits(app.allocator, a.symbol)) |spl| {
|
||||
split_map.put(a.symbol, spl.data) catch {
|
||||
app.allocator.free(spl.data);
|
||||
break;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
state.view = review_view.buildReview(
|
||||
app.allocator,
|
||||
app.io,
|
||||
summary_ptr.*,
|
||||
candle_map,
|
||||
dividend_map,
|
||||
&split_map,
|
||||
pf,
|
||||
cm_ptr.*,
|
||||
acct_map_opt,
|
||||
|
|
@ -2426,6 +2450,7 @@ test "deinitState: cleans up view (leak check)" {
|
|||
summary,
|
||||
&candle_map,
|
||||
null,
|
||||
null,
|
||||
portfolio,
|
||||
cm,
|
||||
null,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ const forecast = @import("../analytics/forecast_evaluation.zig");
|
|||
const timeline = @import("../analytics/timeline.zig");
|
||||
const valuation = @import("../analytics/valuation.zig");
|
||||
const zfin = @import("../root.zig");
|
||||
const sliceCandlesAsOf = @import("../models/candle.zig").sliceCandlesAsOf;
|
||||
const snapshot_model = @import("../models/snapshot.zig");
|
||||
const history = @import("../history.zig");
|
||||
const Date = @import("../Date.zig");
|
||||
|
|
@ -823,10 +824,9 @@ fn buildContextFromParts(
|
|||
cd_value,
|
||||
);
|
||||
|
||||
// Fetch benchmark candles (checks cache first). In historical
|
||||
// mode we slice to `<= as_of` - `performance.trailingReturns`
|
||||
// anchors on the last candle's date, so trimming the tail gives
|
||||
// returns "as of" that date for free.
|
||||
// Fetch benchmark candles (checks cache first). `totalReturns`
|
||||
// truncates to `<= as_of` internally, so historical mode needs no
|
||||
// pre-slicing here.
|
||||
//
|
||||
// Symbols default to SPY/AGG; user can override via
|
||||
// `type::config,benchmark_stock::SYMBOL` and
|
||||
|
|
@ -835,36 +835,50 @@ fn buildContextFromParts(
|
|||
const bond_sym = config.benchmark_bond;
|
||||
const spy_result = svc.getCandles(stock_sym, .{}) catch null;
|
||||
defer if (spy_result) |r| r.deinit();
|
||||
const spy_candles = history.sliceCandlesAsOf(
|
||||
if (spy_result) |r| r.data else &.{},
|
||||
as_of,
|
||||
);
|
||||
const spy_divs = svc.getCachedDividends(alloc, stock_sym);
|
||||
defer if (spy_divs) |d| d.deinit();
|
||||
|
||||
const agg_result = svc.getCandles(bond_sym, .{}) catch null;
|
||||
defer if (agg_result) |r| r.deinit();
|
||||
const agg_candles = history.sliceCandlesAsOf(
|
||||
const agg_divs = svc.getCachedDividends(alloc, bond_sym);
|
||||
defer if (agg_divs) |d| d.deinit();
|
||||
|
||||
// Total return, not the bare adj_close ratio. This path used to
|
||||
// call `performance.trailingReturns` directly, which is the
|
||||
// adj_close series alone - so a stale adjustment basis understated
|
||||
// the benchmark, the portfolio-weighted return, and
|
||||
// `conservative_return`, which drives the Monte Carlo projections.
|
||||
const spy_trailing = performance.totalReturns(
|
||||
if (spy_result) |r| r.data else &.{},
|
||||
if (spy_divs) |d| d.data else null,
|
||||
as_of,
|
||||
);
|
||||
const agg_trailing = performance.totalReturns(
|
||||
if (agg_result) |r| r.data else &.{},
|
||||
if (agg_divs) |d| d.data else null,
|
||||
as_of,
|
||||
);
|
||||
|
||||
const spy_trailing = performance.trailingReturns(spy_candles);
|
||||
const agg_trailing = performance.trailingReturns(agg_candles);
|
||||
|
||||
// Build per-position trailing returns from cached candles, each
|
||||
// optionally truncated to the as-of date. `trailingReturns`
|
||||
// populates `.week` per position; `portfolioWeightedReturns`
|
||||
// aggregates the week the same way as the longer periods.
|
||||
// Build per-position trailing returns from cached candles.
|
||||
// `totalReturns` populates `.week` per position;
|
||||
// `portfolioWeightedReturns` aggregates the week the same way as
|
||||
// the longer periods.
|
||||
var pos_returns: std.ArrayListUnmanaged(benchmark.PositionReturn) = .empty;
|
||||
defer pos_returns.deinit(alloc);
|
||||
for (allocations) |a| {
|
||||
const candles_res = svc.getCachedCandles(alloc, a.symbol) orelse continue;
|
||||
defer candles_res.deinit();
|
||||
const candles = history.sliceCandlesAsOf(candles_res.data, as_of);
|
||||
if (candles.len > 0) {
|
||||
const pos_divs = svc.getCachedDividends(alloc, a.symbol);
|
||||
defer if (pos_divs) |d| d.deinit();
|
||||
if (sliceCandlesAsOf(candles_res.data, as_of).len > 0) {
|
||||
try pos_returns.append(alloc, .{
|
||||
.symbol = a.symbol,
|
||||
.weight = a.weight,
|
||||
.returns = performance.trailingReturns(candles),
|
||||
.returns = performance.totalReturns(
|
||||
candles_res.data,
|
||||
if (pos_divs) |d| d.data else null,
|
||||
as_of,
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -186,8 +186,10 @@ pub const ReviewView = struct {
|
|||
/// - `dividend_map` (optional): symbol -> dividend slices, used to
|
||||
/// compute total-return numbers. Pass `null` to fall back to
|
||||
/// adj_close-derived returns. (Most providers bake dividends into
|
||||
/// adj_close, so the fallback is usually fine - `withDividendFallback`
|
||||
/// picks whichever number is higher per period.)
|
||||
/// adj_close, so the fallback is usually fine, but it inherits any
|
||||
/// staleness in the cached adjustment basis - see
|
||||
/// `CandleMeta.adj_basis`. `performance.totalReturnsMonthEnd` picks
|
||||
/// whichever number is higher per period.)
|
||||
/// - `classifications`: parsed `metadata.srf`
|
||||
/// - `account_map` (optional): parsed `accounts.srf`. When null,
|
||||
/// Tax% is reported as null on every row.
|
||||
|
|
@ -199,6 +201,11 @@ pub fn buildReview(
|
|||
summary: valuation.PortfolioSummary,
|
||||
candle_map: *const std.StringHashMap([]const zfin.Candle),
|
||||
dividend_map: ?*const std.StringHashMap([]const zfin.Dividend),
|
||||
/// Per-symbol splits, newest-first. Travels with `dividend_map`:
|
||||
/// the totals row builds a total-return index from raw `close`,
|
||||
/// which is not split-adjusted, so dividends without splits would
|
||||
/// read a 2:1 split as a -50% month.
|
||||
split_map: ?*const std.StringHashMap([]const zfin.Split),
|
||||
portfolio: zfin.Portfolio,
|
||||
classifications: classification.ClassificationMap,
|
||||
account_map: ?analysis.AccountMap,
|
||||
|
|
@ -233,7 +240,7 @@ pub fn buildReview(
|
|||
const bucket = bucketForSymbol(a.symbol, classifications);
|
||||
const tax_pct = computeTaxPct(a.symbol, portfolio, account_map, as_of);
|
||||
|
||||
const tr_returns = computeTrailingReturns(candles, dividends, as_of);
|
||||
const tr_returns = performance.totalReturnsMonthEnd(candles, dividends, as_of);
|
||||
const tr_risk = if (candles.len > 0) risk.trailingRisk(candles) else risk.TrailingRisk{};
|
||||
|
||||
try rows.append(allocator, .{
|
||||
|
|
@ -257,6 +264,11 @@ pub fn buildReview(
|
|||
.symbol = a.symbol,
|
||||
.candles = candles,
|
||||
.weight = a.weight,
|
||||
// Supplying both switches the synthetic series from
|
||||
// `adj_close` to a total-return index; supplying neither
|
||||
// leaves it on `adj_close`. Never supply just one.
|
||||
.dividends = dividends orelse &.{},
|
||||
.splits = if (split_map) |sm| (sm.get(a.symbol) orelse &.{}) else &.{},
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -583,24 +595,6 @@ fn accountIsKnown(am: analysis.AccountMap, account: []const u8) bool {
|
|||
return false;
|
||||
}
|
||||
|
||||
/// Compute month-end total-return trailing returns. Falls back to
|
||||
/// adj_close-only when no dividend data is available, picking the
|
||||
/// higher value per period via `withDividendFallback`.
|
||||
fn computeTrailingReturns(
|
||||
candles: []const zfin.Candle,
|
||||
dividends: ?[]const zfin.Dividend,
|
||||
as_of: Date,
|
||||
) performance.TrailingReturns {
|
||||
if (candles.len == 0) return .{};
|
||||
|
||||
const adj = performance.trailingReturnsMonthEnd(candles, as_of);
|
||||
if (dividends) |divs| {
|
||||
const div = performance.trailingReturnsMonthEndWithDividends(candles, divs, as_of);
|
||||
return performance.withDividendFallback(div, adj);
|
||||
}
|
||||
return adj;
|
||||
}
|
||||
|
||||
/// Pull the annualized return out of a `PerformanceResult`. For 1Y,
|
||||
/// the period is right at the threshold - `annualizedReturn` returns
|
||||
/// null when the actual span < 0.95 years. We accept that null and
|
||||
|
|
@ -844,58 +838,6 @@ test "sortRows: every numeric SortField variant is reachable via extractFloat" {
|
|||
}
|
||||
}
|
||||
|
||||
test "computeTrailingReturns: empty candles returns empty struct" {
|
||||
const tr = computeTrailingReturns(&.{}, null, Date.fromYmd(2026, 6, 1));
|
||||
try testing.expect(tr.one_year == null);
|
||||
try testing.expect(tr.three_year == null);
|
||||
try testing.expect(tr.five_year == null);
|
||||
try testing.expect(tr.ten_year == null);
|
||||
}
|
||||
|
||||
test "computeTrailingReturns: candles with no dividends uses adj_close path" {
|
||||
// 36 months of 1% monthly growth -> ~12.7% annualized. Without
|
||||
// dividends, this exercises the adj-close-only branch. Even when
|
||||
// the synthetic data isn't enough to land any specific trailing
|
||||
// window (depends on calendar arithmetic against the as_of), the
|
||||
// function shouldn't crash and should return a struct.
|
||||
var candles: [36]zfin.Candle = undefined;
|
||||
var d = Date.fromYmd(2023, 1, 31);
|
||||
for (0..36) |i| {
|
||||
const price: f64 = 100.0 * std.math.pow(f64, 1.01, @as(f64, @floatFromInt(i)));
|
||||
candles[i] = .{ .date = d, .open = price, .high = price, .low = price, .close = price, .adj_close = price, .volume = 1000 };
|
||||
d = d.addDays(30);
|
||||
}
|
||||
const tr = computeTrailingReturns(&candles, null, Date.fromYmd(2026, 1, 31));
|
||||
// We don't assert on which windows populated - that depends on
|
||||
// calendar arithmetic against `as_of` and the candle density.
|
||||
// Coverage of the no-dividends branch (the `else` arm at the
|
||||
// end of `computeTrailingReturns`) is what we want.
|
||||
_ = tr;
|
||||
}
|
||||
|
||||
test "computeTrailingReturns: candles with dividends prefers higher fallback" {
|
||||
// Same shape, but include a dividends slice. The fallback logic
|
||||
// takes whichever produces a higher annualized return per period;
|
||||
// for adj_close already including dividends and an explicit
|
||||
// dividends slice, the helper picks whichever is higher. Either
|
||||
// way the path through `withDividendFallback` is exercised.
|
||||
var candles: [36]zfin.Candle = undefined;
|
||||
var d = Date.fromYmd(2023, 1, 31);
|
||||
for (0..36) |i| {
|
||||
const price: f64 = 100.0 + @as(f64, @floatFromInt(i));
|
||||
candles[i] = .{ .date = d, .open = price, .high = price, .low = price, .close = price, .adj_close = price, .volume = 1000 };
|
||||
d = d.addDays(30);
|
||||
}
|
||||
const divs = [_]zfin.Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2023, 6, 15), .pay_date = Date.fromYmd(2023, 7, 1), .amount = 0.5 },
|
||||
.{ .ex_date = Date.fromYmd(2024, 6, 15), .pay_date = Date.fromYmd(2024, 7, 1), .amount = 0.5 },
|
||||
};
|
||||
const tr = computeTrailingReturns(&candles, &divs, Date.fromYmd(2026, 1, 31));
|
||||
// We don't need to assert exact values - coverage of the
|
||||
// dividend-fallback branch is the goal.
|
||||
_ = tr;
|
||||
}
|
||||
|
||||
test "computeTaxPct: returns null when account_map is null" {
|
||||
var lots = [_]zfin.Lot{
|
||||
.{ .symbol = "VTI", .shares = 100, .open_date = Date.fromYmd(2022, 1, 10), .open_price = 200, .account = "Brokerage" },
|
||||
|
|
@ -1274,6 +1216,7 @@ test "buildReview: dividends_missing separates a failed fetch from a holding tha
|
|||
summary,
|
||||
&candle_map,
|
||||
÷nd_map,
|
||||
null,
|
||||
portfolio,
|
||||
cm,
|
||||
null,
|
||||
|
|
@ -1303,6 +1246,7 @@ test "buildReview: dividends_missing separates a failed fetch from a holding tha
|
|||
summary,
|
||||
&candle_map,
|
||||
null,
|
||||
null,
|
||||
portfolio,
|
||||
cm,
|
||||
null,
|
||||
|
|
@ -1402,6 +1346,7 @@ test "buildReview: end-to-end with testing allocator (leak check)" {
|
|||
summary,
|
||||
&candle_map,
|
||||
null, // no dividend map
|
||||
null, // no split map
|
||||
portfolio,
|
||||
cm,
|
||||
am,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue