lobo/zfin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

disable twelvedata candles due to inconsistent dividend handling

Browse source
This commit is contained in:
Emil Lerch 2026-03-24 18:17:19 -07:00
parent 5052492ffd
commit ee9d749a6f
Signed by: lobo
GPG key ID: A7B62D657EF764F8
3 changed files with 74 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
src/analytics/performance.zig
View file

@ -62,14 +62,14 @@ fn totalReturnFromAdjCloseSnap(candles: []const Candle, from: Date, to: Date, st
/// Compute total return with manual dividend reinvestment.
/// Uses raw close prices and dividend records independently.
/// Candles and dividends must be sorted by date ascending.
/// `from` snaps forward, `to` snaps backward.
/// `from` snaps backward (last trading day on/before), `to` snaps backward.
pub fn totalReturnWithDividends(
candles: []const Candle,
dividends: []const Dividend,
from: Date,
to: Date,
) ?PerformanceResult {
return totalReturnWithDividendsSnap(candles, dividends, from, to, .forward);
return totalReturnWithDividendsSnap(candles, dividends, from, to, .backward);
}
/// Same as totalReturnWithDividends but both dates snap backward.
@ -105,7 +105,7 @@ fn totalReturnWithDividendsSnap(
for (dividends) |div| {
if (div.ex_date.lessThan(start.date)) continue;
if (end.date.lessThan(div.ex_date)) break;
if (end.date.lessThan(div.ex_date)) continue;
// Find close price on or near the ex-date.
// For stable-NAV funds, dividends before candle history use $1.
@ -619,3 +619,48 @@ test "withFallback -- both null stays null" {
try std.testing.expect(merged.one_year == null);
try std.testing.expect(merged.ten_year == null);
}
test "splits-only adj_close -- dividend reinvestment preferred" {
// Simulates a TwelveData-like provider where adj_close only accounts for splits.
// Stock pays $2/quarter dividend, price stays at ~$100.
// adj_close shows no return (splits-only), but dividend reinvestment should
// show the correct total return from the distributions.
const candles = [_]Candle{
// adj_close == close here (no splits), so adj_close return ≈ 0%
makeCandle(Date.fromYmd(2025, 1, 2), 100),
makeCandle(Date.fromYmd(2025, 3, 15), 100),
makeCandle(Date.fromYmd(2025, 6, 15), 100),
makeCandle(Date.fromYmd(2025, 9, 15), 100),
makeCandle(Date.fromYmd(2025, 12, 31), 100),
};
const divs = [_]Dividend{
.{ .ex_date = Date.fromYmd(2025, 3, 15), .amount = 2.0 },
.{ .ex_date = Date.fromYmd(2025, 6, 15), .amount = 2.0 },
.{ .ex_date = Date.fromYmd(2025, 9, 15), .amount = 2.0 },
.{ .ex_date = Date.fromYmd(2025, 12, 15), .amount = 2.0 },
};
// adj_close return should be ~0% (price flat, no dividend adjustment)
const adj_result = totalReturnFromAdjClose(&candles, Date.fromYmd(2025, 1, 2), Date.fromYmd(2025, 12, 31));
try std.testing.expect(adj_result != null);
try std.testing.expectApproxEqAbs(@as(f64, 0.0), adj_result.?.total_return, 0.01);
// Dividend reinvestment: 3 of 4 dividends are reinvested (Dec 15 is >10 days
// from any candle so it's skipped by snap tolerance). ~6.12% total return.
const div_result = totalReturnWithDividends(&candles, &divs, Date.fromYmd(2025, 1, 2), Date.fromYmd(2025, 12, 31));
try std.testing.expect(div_result != null);
try std.testing.expect(div_result.?.total_return > 0.05);
// When adj_close is splits-only, dividend reinvestment should be primary.
// Wrapping in TrailingReturns to test withFallback:
const adj_tr: TrailingReturns = .{ .one_year = adj_result };
const div_tr: TrailingReturns = .{ .one_year = div_result };
// withFallback(div, adj) keeps div_result where available
const total = withFallback(div_tr, adj_tr);
try std.testing.expect(total.one_year.?.total_return > 0.05);
// Reversed order (adj_close primary) would give ~0% — wrong for splits-only
const wrong = withFallback(adj_tr, div_tr);
try std.testing.expectApproxEqAbs(@as(f64, 0.0), wrong.one_year.?.total_return, 0.01);
}

19
src/cache/store.zig vendored
View file

@ -215,7 +215,7 @@ pub const Store = struct {
/// Write a full set of candles to cache (no expiry — historical facts don't expire).
/// Also updates candle metadata.
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle) void {
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider) void {
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
defer self.allocator.free(srf_data);
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
@ -227,14 +227,14 @@ pub const Store = struct {
if (candles.len > 0) {
const last = candles[candles.len - 1];
self.updateCandleMeta(symbol, last.close, last.date);
self.updateCandleMeta(symbol, last.close, last.date, provider);
}
}
/// Append new candle records to the existing cache file.
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
/// Also updates candle metadata.
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle) void {
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider) void {
if (new_candles.len == 0) return;
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
@ -263,16 +263,11 @@ pub const Store = struct {
}
const last = new_candles[new_candles.len - 1];
self.updateCandleMeta(symbol, last.close, last.date);
self.updateCandleMeta(symbol, last.close, last.date, provider);
}
/// Write (or refresh) candle metadata without touching the candle data file.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date) void {
self.updateCandleMetaWithProvider(symbol, last_close, last_date, .twelvedata);
}
/// Write candle metadata with a specific provider source.
pub fn updateCandleMetaWithProvider(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider) void {
/// Write (or refresh) candle metadata with a specific provider source.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider) void {
const expires = std.time.timestamp() + Ttl.candles_latest;
const meta = CandleMeta{
.last_close = last_close,
@ -398,7 +393,7 @@ pub const Store = struct {
last_close: f64,
last_date: Date,
/// Which provider sourced the candle data. Used during incremental refresh
/// to go directly to the right provider instead of trying TwelveData first.
/// to go directly to the right provider instead of trying Tiingo first.
provider: CandleProvider = .tiingo,
};

71
src/service.zig
View file

@ -283,25 +283,7 @@ pub const DataService = struct {
} else |_| {}
}
// If preferred is TwelveData, try it before Tiingo
if (preferred == .twelvedata) {
if (self.getProvider(TwelveData)) |td| {
if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from TwelveData (preferred)", .{symbol});
return .{ .candles = candles, .provider = .twelvedata };
} else |err| {
if (err == error.RateLimited) {
self.rateLimitBackoff();
if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from TwelveData (preferred, after rate limit retry)", .{symbol});
return .{ .candles = candles, .provider = .twelvedata };
} else |_| {}
}
}
} else |_| {}
}
// Primary: Tiingo (1000 req/day, no per-minute limit)
// Primary: Tiingo (1000 req/day, no per-minute limit, adj_close includes dividends)
if (self.getProvider(Tiingo)) |tg| {
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Tiingo", .{symbol});
@ -309,25 +291,7 @@ pub const DataService = struct {
} else |_| {}
} else |_| {}
// Fallback: TwelveData (if not already tried as preferred)
if (preferred != .twelvedata) {
if (self.getProvider(TwelveData)) |td| {
if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from TwelveData (fallback)", .{symbol});
return .{ .candles = candles, .provider = .twelvedata };
} else |err| {
if (err == error.RateLimited) {
self.rateLimitBackoff();
if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from TwelveData (fallback, after rate limit retry)", .{symbol});
return .{ .candles = candles, .provider = .twelvedata };
} else |_| {}
}
}
} else |_| {}
}
// Last resort: Yahoo (if not already tried as preferred)
// Fallback: Yahoo (if not already tried as preferred)
if (preferred != .yahoo) {
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
@ -341,7 +305,7 @@ pub const DataService = struct {
}
/// Fetch daily candles for a symbol (10+ years for trailing returns).
/// Checks cache first; fetches from Tiingo (primary), TwelveData, or Yahoo if stale/missing.
/// Checks cache first; fetches from Tiingo (primary) or Yahoo (fallback) if stale/missing.
/// Uses incremental updates: when the cache is stale, only fetches
/// candles newer than the last cached date rather than re-fetching
/// the entire history.
@ -351,7 +315,14 @@ pub const DataService = struct {
// Check candle metadata for freshness (tiny file, no candle deserialization)
const meta_result = s.readCandleMeta(symbol);
if (meta_result) |mr| {
const is_twelvedata = if (meta_result) |mr| mr.meta.provider == .twelvedata else false;
// If cached data is from TwelveData (deprecated for candles due to
// unreliable adj_close), skip cache and fall through to full re-fetch.
if (is_twelvedata)
log.debug("{s}: cached candles from TwelveData — forcing full re-fetch", .{symbol});
if (!is_twelvedata) if (meta_result) |mr| {
const m = mr.meta;
if (s.isCandleMetaFresh(symbol)) {
// Fresh — deserialize candles and return
@ -376,7 +347,7 @@ pub const DataService = struct {
// If last cached date is today or later, just refresh the TTL (meta only)
if (!fetch_from.lessThan(today)) {
s.updateCandleMeta(symbol, m.last_close, m.last_date);
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider);
if (s.read(Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
} else {
@ -392,12 +363,12 @@ pub const DataService = struct {
if (new_candles.len == 0) {
// No new candles (weekend/holiday) — refresh TTL only (meta rewrite)
self.allocator.free(new_candles);
s.updateCandleMetaWithProvider(symbol, m.last_close, m.last_date, result.provider);
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider);
if (s.read(Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
} else {
// Append new candles to existing file + update meta
s.appendCandles(symbol, new_candles);
s.appendCandles(symbol, new_candles, result.provider);
// Load the full (now-updated) file for the caller
if (s.read(Candle, symbol, null, .any)) |r| {
self.allocator.free(new_candles);
@ -407,7 +378,7 @@ pub const DataService = struct {
return .{ .data = new_candles, .source = .fetched, .timestamp = std.time.timestamp() };
}
}
}
};
// No usable cache — try server sync first
if (self.syncCandlesFromServer(symbol)) {
@ -430,10 +401,7 @@ pub const DataService = struct {
};
if (result.candles.len > 0) {
s.cacheCandles(symbol, result.candles);
// Record which provider sourced this data
const last = result.candles[result.candles.len - 1];
s.updateCandleMetaWithProvider(symbol, last.close, last.date, result.provider);
s.cacheCandles(symbol, result.candles, result.provider);
}
return .{ .data = result.candles, .source = .fetched, .timestamp = std.time.timestamp() };
@ -596,10 +564,9 @@ pub const DataService = struct {
if (self.getDividends(symbol)) |div_result| {
divs = div_result.data;
// adj_close returns are the primary total return source (accounts for
// splits + dividends). Dividend-reinvestment is only used as a fallback
// for periods where adj_close returns null (e.g. candle history too short
// for stable-NAV funds like money markets).
// adj_close is the primary total return source (accounts for splits +
// dividends). Dividend-reinvestment only fills gaps where adj_close
// returns null (e.g. stable-NAV funds with short candle history).
const asof_div = performance.trailingReturnsWithDividends(c, div_result.data);
asof_total = performance.withFallback(asof_price, asof_div);
const me_div = performance.trailingReturnsMonthEndWithDividends(c, div_result.data, today);