From d9f2e8404bf30fa445142f63d1805e20a4d2f64c Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 20 May 2026 15:20:44 -0700 Subject: [PATCH] allow tiingo to merge in dividends/splits not captured by primary provider --- README.md | 51 ++--- src/cache/store.zig | 435 ++++++++++++++++++++++++++++++++++++++- src/providers/tiingo.zig | 349 +++++++++++++++++++++++++++---- src/service.zig | 100 +++++++-- 4 files changed, 848 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index eccb5a6..b75c2fe 100644 --- a/README.md +++ b/README.md @@ -45,30 +45,31 @@ zfin aggregates data from multiple free-tier APIs. Each provider is used for the ### Tiingo -**Used for:** daily candles (primary provider for all symbols). +**Used for:** daily candles (primary provider for all symbols), supplementary dividend and split data. - Endpoint: `https://api.tiingo.com/tiingo/daily/{symbol}/prices` - Free tier: 1,000 requests per day, no per-minute restriction. - Covers stocks, ETFs, and mutual funds. Mutual fund NAVs are available after midnight ET. -- Candles are fetched with a 10-year + 60-day lookback window for trailing return calculations. -- Returns split-adjusted prices with `adjClose` for dividend-adjusted values. +- Candles are fetched with a fixed 2000-01-01 start date so the cache supports `--as-of` projections back to the earliest imported portfolio data (typically 2014) with full 10Y trailing-return windows. +- The same response carries per-row `divCash` and `splitFactor`. We extract these as a free side benefit and merge them into the dividend/split caches alongside Polygon's primary view -- this rescues entries Polygon's reference endpoints miss (e.g. SPYM's 2017-10-16 4:1 split). ### TwelveData -**Used for:** candle fallback (when Tiingo fails), real-time quotes (fallback after Yahoo). +**Used for:** real-time quote fallback (after Yahoo). -- Endpoint: `https://api.twelvedata.com/time_series` and `/quote` -- Free tier: 8 API credits per minute, 800 per day. Each symbol in a request costs 1 credit. -- Mutual fund NAV updates can lag by a full trading day compared to Tiingo. +- Endpoint: `https://api.twelvedata.com/quote` +- Free tier: 8 API credits per minute, 800 per day. +- TwelveData was previously used for candles but is no longer in the candle pipeline -- its `adj_close` values were unreliable for split-adjustment math. Yahoo is the candle fallback now. ### Polygon -**Used for:** dividend and stock splits information, both historical and upcoming. +**Used for:** dividend and stock split data, both historical and forward-looking. - Endpoints: `https://api.polygon.io/v3/reference/dividends` and `/v3/reference/splits` -- Free tier: 5 requests per minute, unlimited daily. Full historical dividend/split data. +- Free tier: 5 requests per minute, unlimited daily. Full historical data. - Dividend endpoint uses cursor-based pagination (automatically followed). -- Provides dividend type classification (regular, special, supplemental). +- Provides dividend type classification (regular, special, supplemental) and richer metadata than Tiingo (`pay_date`, `record_date`, `currency`). +- Polygon is **primary** for dividends and splits because it carries forward-looking declared events (e.g. ARCC's next ex-dividend date several months out) that Tiingo's price-series response cannot provide. Tiingo merges in supplementary entries for historical events Polygon's reference endpoints occasionally miss. ### CBOE @@ -101,7 +102,7 @@ Set keys as environment variables or in a `.env` file (searched in the executabl ```bash TIINGO_API_KEY=your_key # Required for candles (primary provider) -TWELVEDATA_API_KEY=your_key # Candle fallback, quote fallback +TWELVEDATA_API_KEY=your_key # Quote fallback (after Yahoo) POLYGON_API_KEY=your_key # Required for dividends/splits (total returns) FMP_API_KEY=your_key # Required for earnings data ALPHAVANTAGE_API_KEY=your_key # Required for ETF profiles @@ -111,13 +112,13 @@ The cache directory defaults to `~/.cache/zfin` and can be overridden with `ZFIN Not all keys are required. Without a key, the corresponding data simply won't be available: -| Key | Without it | -|------------------------|--------------------------------------------------------------------| -| `TIINGO_API_KEY` | Candles fall back to TwelveData, then Yahoo | -| `TWELVEDATA_API_KEY` | No candle fallback after Tiingo, no quote fallback after Yahoo | -| `POLYGON_API_KEY` | No dividends -- trailing returns show price-only (no total return) | -| `FMP_API_KEY` | No earnings data (tab disabled) | -| `ALPHAVANTAGE_API_KEY` | No ETF profiles | +| Key | Without it | +|------------------------|-------------------------------------------------------------------------------------| +| `TIINGO_API_KEY` | Candles fall back to Yahoo only; some symbols (especially mutual funds) won't work | +| `TWELVEDATA_API_KEY` | No quote fallback after Yahoo | +| `POLYGON_API_KEY` | No forward-looking dividends; trailing total returns may use only Tiingo's view | +| `FMP_API_KEY` | No earnings data (tab disabled) | +| `ALPHAVANTAGE_API_KEY` | No ETF profiles | CBOE options require no API key. @@ -276,7 +277,7 @@ The TUI has eight tabs: Portfolio, Analysis, Projections, History, Quote, Perfor **Quote** -- current price, OHLCV, daily change, and a 60-day ASCII chart with recent history table. -**Performance** -- trailing returns using two methodologies (as-of-date and month-end), matching Morningstar's "Trailing Returns" and "Performance" pages respectively. Shows price-only and total return (with dividend reinvestment) when Polygon data is available. Also shows risk metrics (volatility, Sharpe ratio, max drawdown). +**Performance** -- trailing returns using two methodologies (as-of-date and month-end), matching Morningstar's "Trailing Returns" and "Performance" pages respectively. Shows price-only and total return (with dividend reinvestment) using whichever dividend data is available -- Polygon (richer metadata, forward-looking entries) and Tiingo (extracted from candle responses, historical only) are merged. Also shows risk metrics (volatility, Sharpe ratio, max drawdown). **Earnings** -- historical and upcoming earnings events with EPS estimate/actual, surprise amount and percentage. Future events are dimmed. Tab is disabled for ETFs. @@ -415,7 +416,7 @@ security_type::watch,symbol::TSLA ### Security types -- **stock** (default) -- Stocks, ETFs, and mutual funds. Prices are fetched from Tiingo (primary), TwelveData, or Yahoo (fallbacks). Positions are aggregated by symbol and shown with gain/loss. +- **stock** (default) -- Stocks, ETFs, and mutual funds. Prices are fetched from Tiingo (primary) or Yahoo (candle fallback). Positions are aggregated by symbol and shown with gain/loss. - **option** -- Option contracts. Shown in a separate "Options" section. Shares can be negative for short positions. - **cd** -- Certificates of deposit. Shown sorted by maturity date with rate and face value. - **cash** -- Cash, money market, and settlement balances. Shown grouped by account with optional notes. @@ -426,7 +427,7 @@ security_type::watch,symbol::TSLA For stock lots, prices are resolved in this order: -1. **Live API** -- Latest close from cached candles (Tiingo/TwelveData/Yahoo) +1. **Live API** -- Latest close from cached candles (Tiingo, with Yahoo as candle fallback) 2. **Manual price** -- `price::` field on the lot (for securities without API coverage, e.g. 401k CIT share classes) 3. **Average cost** -- Falls back to the position's `open_price` as a last resort @@ -866,13 +867,13 @@ src/ classification.zig Classification metadata parser quote.zig Real-time quote data providers/ - tiingo.zig Tiingo: daily candles (primary) - twelvedata.zig TwelveData: candles (fallback), quotes (fallback) - polygon.zig Polygon: dividends, splits + tiingo.zig Tiingo: daily candles (primary), supplementary div/split merge + twelvedata.zig TwelveData: quote fallback + polygon.zig Polygon: dividends, splits (primary, with forward-looking entries) fmp.zig FMP: earnings (actuals + estimates) cboe.zig CBOE: options chains (no API key) alphavantage.zig Alpha Vantage: ETF profiles, company overview - yahoo.zig Yahoo Finance: quotes (primary), candles (last resort) + yahoo.zig Yahoo Finance: quotes (primary), candles (Tiingo fallback) openfigi.zig OpenFIGI: CUSIP to ticker lookup analytics/ indicators.zig SMA, Bollinger Bands, RSI diff --git a/src/cache/store.zig b/src/cache/store.zig index 545b9d8..d484922 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -196,6 +196,13 @@ pub const Store = struct { /// Serialize data and write to cache with the given TTL. /// Accepts a slice for most types, or a single struct for EtfProfile. + /// + /// For `Dividend` and `Split`, this dispatches to `writeMerged`, + /// which performs sorted-union-with-existing semantics rather than + /// a clean overwrite. Both Tiingo's full-history view and + /// Polygon's targeted fetches converge to the same on-disk union + /// regardless of write order, and forward-looking entries from + /// Polygon are preserved across Tiingo refreshes. pub fn write( self: *Store, comptime T: type, @@ -203,6 +210,26 @@ pub const Store = struct { items: DataFor(T), ttl: i64, ) void { + self.writeWithSource(T, symbol, items, ttl, null); + } + + /// Same as `write` but lets the caller attribute new entries to a + /// named source (e.g. `"tiingo"`). The source name appears in the + /// `info(cache)` log line emitted by `writeMerged` when a + /// previously-unseen dividend or split lands in the cache. For + /// types that don't go through the merge path, the hint is unused. + pub fn writeWithSource( + self: *Store, + comptime T: type, + symbol: []const u8, + items: DataFor(T), + ttl: i64, + source_hint: ?[]const u8, + ) void { + if (T == Dividend or T == Split) { + self.writeMerged(T, symbol, items, ttl, source_hint); + return; + } const expires = std.Io.Timestamp.now(self.io, .real).toSeconds() + ttl; const data_type = dataTypeFor(T); if (T == EtfProfile) { @@ -237,6 +264,117 @@ pub const Store = struct { }; } + /// Sorted-union write for `Dividend` and `Split`. Reads the + /// existing cache file, adds any items from `incoming` whose + /// date key isn't already present, sorts the union descending + /// by date, and writes the result. If nothing new came in, the + /// existing file is left untouched (no mtime bump, no I/O). + /// + /// Existing entries always win on key collision: Polygon's + /// dividend records carry richer metadata (`pay_date`, + /// `record_date`, `type`, `currency`) than Tiingo's, so once + /// Polygon has supplied an ex_date we don't want a later Tiingo + /// write to overwrite it with a sparser record. The merge + /// preserves whichever entry landed first for any given date. + /// + /// Each newly-added entry triggers an `info(cache)` log line so + /// the user is alerted when a supplementary source (Tiingo, + /// usually) discovers a corporate action the primary source + /// (Polygon) missed. The `source_hint` argument, when present, + /// names the source in that log line. + fn writeMerged( + self: *Store, + comptime T: type, + symbol: []const u8, + incoming: []const T, + ttl: i64, + source_hint: ?[]const u8, + ) void { + comptime std.debug.assert(T == Dividend or T == Split); + + // Read existing entries (any freshness; we want the union of + // what's on disk, not just fresh data). + const existing_result = self.read(T, symbol, null, .any); + const existing: []const T = if (existing_result) |r| r.data else &.{}; + defer if (existing_result != null) { + if (comptime @hasDecl(T, "deinit")) { + for (existing) |item| item.deinit(self.allocator); + } + self.allocator.free(existing); + }; + + // Build the union. Start with existing entries, then append + // any incoming entry whose key isn't already present. + var merged: std.ArrayList(T) = .empty; + defer merged.deinit(self.allocator); + merged.appendSlice(self.allocator, existing) catch return; + + var added: usize = 0; + for (incoming) |item| { + if (containsKey(T, merged.items, mergeKey(T, item))) continue; + merged.append(self.allocator, item) catch return; + added += 1; + logSupplied(T, symbol, item, source_hint); + } + + if (added == 0) { + // Nothing new — leave the file untouched. This is the + // common case for repeated Polygon/Tiingo refreshes. + return; + } + + // Sort descending by date (newest first), matching the + // existing on-disk convention. + std.mem.sort(T, merged.items, {}, lessByDateDesc(T)); + + // Serialize via the same generic path as `write` for + // non-merged types, but write the union we just built. + const expires = std.Io.Timestamp.now(self.io, .real).toSeconds() + ttl; + const data_type = dataTypeFor(T); + const srf_data = serializeWithMeta(T, self.io, self.allocator, merged.items, .{ .expires = expires }) catch |err| { + log.warn("{s}: failed to serialize {s}: {s}", .{ symbol, @tagName(data_type), @errorName(err) }); + return; + }; + defer self.allocator.free(srf_data); + self.writeRaw(symbol, data_type, srf_data) catch |err| { + log.warn("{s}: failed to write {s} to cache: {s}", .{ symbol, @tagName(data_type), @errorName(err) }); + }; + } + + fn mergeKey(comptime T: type, item: T) i32 { + if (T == Dividend) return item.ex_date.days; + if (T == Split) return item.date.days; + @compileError("mergeKey only defined for Dividend and Split"); + } + + fn containsKey(comptime T: type, items: []const T, key: i32) bool { + for (items) |it| { + if (mergeKey(T, it) == key) return true; + } + return false; + } + + fn lessByDateDesc(comptime T: type) fn (void, T, T) bool { + return struct { + fn lt(_: void, a: T, b: T) bool { + return mergeKey(T, a) > mergeKey(T, b); + } + }.lt; + } + + fn logSupplied(comptime T: type, symbol: []const u8, item: T, source_hint: ?[]const u8) void { + const source = source_hint orelse "fetch"; + if (T == Dividend) { + log.info("{s}: {s} supplied dividend ex_date {f} amount ${d:.4}", .{ + symbol, source, item.ex_date, item.amount, + }); + } else if (T == Split) { + log.info("{s}: {s} supplied split {f} {d}:{d}", .{ + symbol, source, item.date, @as(u64, @intFromFloat(item.numerator)), @as(u64, @intFromFloat(item.denominator)), + }); + } + } + // ── Candle-specific API ────────────────────────────────────── /// Write a full set of candles to cache (no expiry — historical facts don't expire). @@ -673,9 +811,23 @@ pub const Store = struct { pub const CandleMeta = struct { last_close: f64, last_date: Date, - /// Which provider sourced the candle data. Used during incremental refresh - /// to go directly to the right provider instead of trying Tiingo first. - provider: CandleProvider = .tiingo, + /// Which provider sourced the candle data. **No default + /// value on purpose** — SRF auto-elides fields whose value + /// equals their default, which would hide the provider line + /// when it equaled the implicit default. We want every cache + /// file to record its provider explicitly so cache inspection + /// can always answer "where did this come from?". Construction + /// sites must pass the provider explicitly. + /// + /// Cache compatibility: pre-2026-05 caches that elided the + /// provider field will fail to deserialize after this change + /// (SRF returns FieldNotFoundOnFieldWithoutDefaultValue). + /// `readCandleMeta` swallows the error and returns null, + /// making the symbol look like a cache miss — `getCandles` + /// then triggers a fresh fetch via `populateAllFromTiingo`, + /// which writes a new meta file with the provider explicit. + /// The wipe happens naturally on first use post-upgrade. + provider: CandleProvider, /// Consecutive transient failure count for the primary provider (Tiingo). /// Incremented on ServerError; reset to 0 on success. When >= 3, the /// symbol is degraded to a fallback provider until Tiingo recovers. @@ -683,8 +835,21 @@ pub const Store = struct { }; pub const CandleProvider = enum { + /// Legacy: candles were sourced from TwelveData. No new + /// writes produce this value (TwelveData was demoted in an + /// earlier change because its `adj_close` was unreliable). + /// Cache reads still recognize the value for backwards + /// compatibility. twelvedata, + /// Legacy: candles were sourced from Yahoo Finance. No new + /// writes produce this value (Yahoo was removed from the + /// candle pipeline in the 2026-05 audit; Yahoo is still used + /// for `getQuote` real-time prices but not for historical + /// candles). Cache reads still recognize the value for + /// backwards compatibility. yahoo, + /// Active: candles sourced from Tiingo. The only value + /// produced by current writes. tiingo, pub fn fromString(s: []const u8) CandleProvider { @@ -851,6 +1016,11 @@ pub const Store = struct { return aw.toOwnedSlice(); } + /// Serialize CandleMeta to its SRF on-disk representation. + /// Uses SRF's generic field emission. Because `CandleMeta` + /// declares no default for `provider`, every meta file emits + /// the provider line explicitly — cache inspection can always + /// answer "where did this come from?". fn serializeCandleMeta(io: std.Io, allocator: std.mem.Allocator, meta: CandleMeta, options: srf.FormatOptions) ![]const u8 { var aw: std.Io.Writer.Allocating = .init(allocator); errdefer aw.deinit(); @@ -1186,6 +1356,195 @@ test "split serialize/deserialize round-trip" { try std.testing.expectApproxEqAbs(@as(f64, 7), parsed[1].numerator, 0.001); } +test "writeMerged Dividend: empty cache writes input sorted descending" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + // Intentionally pass entries out of order — writeMerged must sort. + var incoming = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50 }, + .{ .ex_date = Date.fromYmd(2024, 8, 15), .amount = 0.55 }, + .{ .ex_date = Date.fromYmd(2024, 2, 15), .amount = 0.48 }, + }; + s.write(Dividend, "TEST", incoming[0..], Ttl.dividends); + + const result = s.read(Dividend, "TEST", null, .any) orelse return error.NoCache; + defer allocator.free(result.data); + defer for (result.data) |d| d.deinit(allocator); + + try std.testing.expectEqual(@as(usize, 3), result.data.len); + try std.testing.expect(result.data[0].ex_date.eql(Date.fromYmd(2024, 8, 15))); + try std.testing.expect(result.data[1].ex_date.eql(Date.fromYmd(2024, 5, 15))); + try std.testing.expect(result.data[2].ex_date.eql(Date.fromYmd(2024, 2, 15))); +} + +test "writeMerged Dividend: existing entries preserved on key collision" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + // Initial write: rich entry from "Polygon" with metadata. + var initial = [_]Dividend{ + .{ + .ex_date = Date.fromYmd(2024, 5, 15), + .pay_date = Date.fromYmd(2024, 6, 1), + .amount = 0.50, + .type = .regular, + }, + }; + s.write(Dividend, "TEST", initial[0..], Ttl.dividends); + + // Second write: same ex_date, sparser entry (Tiingo-style: no pay_date, no type). + // Existing entry should win. + var incoming = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.99, .type = .unknown }, + }; + s.write(Dividend, "TEST", incoming[0..], Ttl.dividends); + + const result = s.read(Dividend, "TEST", null, .any) orelse return error.NoCache; + defer allocator.free(result.data); + defer for (result.data) |d| d.deinit(allocator); + + try std.testing.expectEqual(@as(usize, 1), result.data.len); + // Original Polygon-style amount (0.50) must remain — Tiingo's 0.99 must not overwrite. + try std.testing.expectApproxEqAbs(@as(f64, 0.50), result.data[0].amount, 0.001); + try std.testing.expect(result.data[0].pay_date != null); + try std.testing.expectEqual(DividendType.regular, result.data[0].type); +} + +test "writeMerged Dividend: union sorted desc, new entry added" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + var initial = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50, .type = .regular }, + .{ .ex_date = Date.fromYmd(2024, 2, 15), .amount = 0.48, .type = .regular }, + }; + s.write(Dividend, "TEST", initial[0..], Ttl.dividends); + + // New ex_date that wasn't already present. + var incoming = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2024, 8, 15), .amount = 0.55 }, + }; + s.write(Dividend, "TEST", incoming[0..], Ttl.dividends); + + const result = s.read(Dividend, "TEST", null, .any) orelse return error.NoCache; + defer allocator.free(result.data); + defer for (result.data) |d| d.deinit(allocator); + + try std.testing.expectEqual(@as(usize, 3), result.data.len); + try std.testing.expect(result.data[0].ex_date.eql(Date.fromYmd(2024, 8, 15))); + try std.testing.expect(result.data[1].ex_date.eql(Date.fromYmd(2024, 5, 15))); + try std.testing.expect(result.data[2].ex_date.eql(Date.fromYmd(2024, 2, 15))); +} + +test "writeMerged Dividend: no-op when nothing new" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + var initial = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50, .type = .regular }, + }; + s.write(Dividend, "TEST", initial[0..], Ttl.dividends); + + // Capture file mtime before second (no-op) write. + const path = try std.fs.path.join(allocator, &.{ dir_path, "TEST", "dividends.srf" }); + defer allocator.free(path); + const stat_before = try std.Io.Dir.cwd().statFile(io, path, .{}); + + // Sleep briefly so mtime resolution can detect a write if one happens. + std.Io.sleep(io, std.Io.Duration.fromMilliseconds(20), .awake) catch {}; + + // Same incoming entry — nothing new, should not rewrite. + var repeat = [_]Dividend{ + .{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50, .type = .regular }, + }; + s.write(Dividend, "TEST", repeat[0..], Ttl.dividends); + + const stat_after = try std.Io.Dir.cwd().statFile(io, path, .{}); + try std.testing.expectEqual(stat_before.mtime, stat_after.mtime); +} + +test "writeMerged Split: SPYM-style supplementary entry added" { + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + // Polygon's view: empty (the bug case — Polygon doesn't carry SPYM's 2017 split). + var initial = [_]Split{}; + s.write(Split, "SPYM", initial[0..], Ttl.splits); + + // Tiingo supplements with the 2017 4:1 split. + var tiingo_view = [_]Split{ + .{ .date = Date.fromYmd(2017, 10, 16), .numerator = 4, .denominator = 1 }, + }; + s.write(Split, "SPYM", tiingo_view[0..], Ttl.splits); + + const result = s.read(Split, "SPYM", null, .any) orelse return error.NoCache; + defer allocator.free(result.data); + + try std.testing.expectEqual(@as(usize, 1), result.data.len); + try std.testing.expect(result.data[0].date.eql(Date.fromYmd(2017, 10, 16))); + try std.testing.expectApproxEqAbs(@as(f64, 4), result.data[0].numerator, 0.001); +} + +test "writeMerged Split: forward-looking Polygon entry preserved across Tiingo refresh" { + // Simulates the ARCC-like case for splits: Polygon writes a + // forward-looking entry; a later Tiingo write must not erase it. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var s = Store.init(io, allocator, dir_path); + // Polygon initial: includes a forward-looking entry. + var polygon_view = [_]Split{ + .{ .date = Date.fromYmd(2026, 12, 1), .numerator = 2, .denominator = 1 }, + .{ .date = Date.fromYmd(2020, 8, 31), .numerator = 4, .denominator = 1 }, + }; + s.write(Split, "TEST", polygon_view[0..], Ttl.splits); + + // Tiingo refresh: only knows about historical entries (its own data + // doesn't include forward-looking yet-to-occur splits). + var tiingo_view = [_]Split{ + .{ .date = Date.fromYmd(2020, 8, 31), .numerator = 4, .denominator = 1 }, + }; + s.write(Split, "TEST", tiingo_view[0..], Ttl.splits); + + const result = s.read(Split, "TEST", null, .any) orelse return error.NoCache; + defer allocator.free(result.data); + + // Both entries must remain — Polygon's forward-looking entry survives. + try std.testing.expectEqual(@as(usize, 2), result.data.len); + try std.testing.expect(result.data[0].date.eql(Date.fromYmd(2026, 12, 1))); + try std.testing.expect(result.data[1].date.eql(Date.fromYmd(2020, 8, 31))); +} + test "portfolio serialize/deserialize round-trip" { const allocator = std.testing.allocator; // Today is after the lots' open_dates and after the one close_date, @@ -1592,14 +1951,82 @@ test "Store init creates valid store" { try std.testing.expectEqualStrings("/tmp/zfin-test", store.cache_dir); } -test "CandleMeta default provider is tiingo" { +test "CandleMeta has no default provider (must be set explicitly)" { + // Regression: pre-2026-05 the model had `provider: CandleProvider = .tiingo` + // which caused SRF to elide the field when it equaled the default, + // hiding the provider in cache inspection. We removed the default + // so every cache file records its provider explicitly. + // + // This test confirms the field is required at construction. If + // someone re-adds a default later, this test fails to compile. const meta = Store.CandleMeta{ .last_close = 100.0, .last_date = Date.fromYmd(2024, 1, 1), + .provider = .tiingo, }; try std.testing.expectEqual(Store.CandleProvider.tiingo, meta.provider); } +test "serializeCandleMeta unconditionally emits provider field" { + // Regression: SRF auto-elision used to hide `provider::tiingo` + // when it equaled the model default. With the default removed + // from the struct, SRF emits the field unconditionally. + const allocator = std.testing.allocator; + const meta = Store.CandleMeta{ + .last_close = 100.0, + .last_date = Date.fromYmd(2024, 1, 1), + .provider = .tiingo, + }; + const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 }); + defer allocator.free(data); + + try std.testing.expect(std.mem.indexOf(u8, data, "provider::tiingo") != null); + try std.testing.expect(std.mem.indexOf(u8, data, "last_close:num:100") != null); + try std.testing.expect(std.mem.indexOf(u8, data, "last_date::2024-01-01") != null); + try std.testing.expect(std.mem.indexOf(u8, data, "#!expires=1234567890") != null); +} + +test "serializeCandleMeta round-trips through deserializeCandleMeta" { + const allocator = std.testing.allocator; + const meta = Store.CandleMeta{ + .last_close = 42.57, + .last_date = Date.fromYmd(2026, 5, 19), + .provider = .tiingo, + .fail_count = 2, + }; + const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 }); + defer allocator.free(data); + + const parsed = try Store.deserializeCandleMeta(allocator, data); + try std.testing.expectApproxEqAbs(@as(f64, 42.57), parsed.last_close, 0.001); + try std.testing.expect(parsed.last_date.eql(Date.fromYmd(2026, 5, 19))); + try std.testing.expectEqual(Store.CandleProvider.tiingo, parsed.provider); + try std.testing.expectEqual(@as(u8, 2), parsed.fail_count); +} + +test "deserializeCandleMeta fails on old cache that elided provider field" { + // Pre-2026-05 cache files elided `provider::tiingo` when it + // equaled the model default. Those caches no longer deserialize + // (model has no default for provider). The graceful handling is + // upstream: `readCandleMeta` swallows the deserialization error + // and returns null, which makes `getCandles` treat it as a cache + // miss and trigger a fresh fetch via `populateAllFromTiingo`. + // The new fetch writes a meta file with the provider explicit. + // + // This test documents the failure mode and confirms it's not a + // silent corruption — the caller gets an error, not stale data. + const allocator = std.testing.allocator; + const old_format = + \\#!srfv1 + \\#!expires=1779384748 + \\#!created=1779299248 + \\last_close:num:298.97,last_date::2026-05-19 + \\ + ; + const result = Store.deserializeCandleMeta(allocator, old_format); + try std.testing.expectError(error.InvalidData, result); +} + // ── writeRaw / appendRaw atomicity ─────────────────────────── // // A concurrent reader hitting a cache file mid-write must never see a diff --git a/src/providers/tiingo.zig b/src/providers/tiingo.zig index 3be083a..32c821c 100644 --- a/src/providers/tiingo.zig +++ b/src/providers/tiingo.zig @@ -1,21 +1,71 @@ -//! Tiingo provider -- official REST API for end-of-day prices. +//! Tiingo provider -- official REST API for end-of-day prices and corporate actions. //! //! Free tier: 1,000 requests/day, no per-minute restriction. //! Covers stocks, ETFs, and mutual funds with same-day NAV updates //! (mutual fund NAVs available after midnight ET). //! //! API docs: https://www.tiingo.com/documentation/end-of-day +//! +//! ## Role in the data pipeline +//! +//! Tiingo is the **primary candle provider**. Yahoo is the fallback +//! when Tiingo can't serve a symbol. Tiingo's `/daily//prices` +//! response also carries per-row `divCash` and `splitFactor`, which +//! we extract during candle parsing as a free side benefit — the +//! candle, dividend, and split data all come from a single HTTP call. +//! +//! For dividends and splits the **primary source is Polygon**, not +//! Tiingo. Polygon's dedicated corporate-actions endpoints carry +//! forward-looking declared events (e.g. ARCC's next ex-dividend +//! date several months out) that Tiingo's price-series response +//! cannot provide — Tiingo only reports events that have already +//! affected a price bar. Polygon also carries richer metadata per +//! dividend (`pay_date`, `record_date`, `type`, `currency`). +//! +//! Tiingo's dividend/split contribution is **supplementary**. The +//! `populateAllFromTiingo` orchestration in `service.zig` writes +//! Tiingo's view through `cache.Store.writeWithSource(..., "tiingo")`, +//! which dispatches to the sorted-union merge primitive. Polygon's +//! existing entries in `dividends.srf` / `splits.srf` are preserved +//! on key collision; Tiingo entries that name new ex_dates / split +//! dates are merged in and logged at `info(cache)` level. +//! +//! The canonical case where Tiingo's supplementary view rescues the +//! cache is SPYM's 2017-10-16 4:1 split — present in Tiingo's +//! historical bars but absent from Polygon's splits endpoint. Without +//! the merge, SPYM's 10Y price-only return would be off by ~14pp. +//! +//! ## Tiingo dividend records carry less metadata +//! +//! Tiingo only emits `divCash` (the cash amount) per dividend event. +//! When Tiingo merges a previously-unseen ex_date into the cache, +//! `pay_date`, `record_date`, `type`, and `currency` will be `null` +//! / `.unknown`. The total-return calculation only needs `ex_date` +//! and `amount`, both of which Tiingo provides; `divs.zig` +//! gracefully handles missing display fields. const std = @import("std"); const http = @import("../net/http.zig"); const Date = @import("../Date.zig"); const Candle = @import("../models/candle.zig").Candle; +const Dividend = @import("../models/dividend.zig").Dividend; +const Split = @import("../models/split.zig").Split; const json_utils = @import("json_utils.zig"); const optFloat = json_utils.optFloat; const jsonStr = json_utils.jsonStr; const base_url = "https://api.tiingo.com/tiingo/daily"; +/// Combined fetch result: candles, dividends, and splits parsed from +/// a single `/daily//prices` response. Caller owns all three +/// slices and must free them (and `Dividend.deinit` each entry for +/// the currency string). +pub const CandleAndCorporateActions = struct { + candles: []Candle, + dividends: []Dividend, + splits: []Split, +}; + pub const Tiingo = struct { client: http.Client, allocator: std.mem.Allocator, @@ -33,15 +83,17 @@ pub const Tiingo = struct { self.client.deinit(); } - /// Fetch daily candles for a symbol between two dates (inclusive). - /// Returns candles sorted oldest-first. - pub fn fetchCandles( + /// Fetch candles, dividends, and splits in one HTTP call. This is + /// the primary provider entry point — the three convenience + /// methods below all call this and free the slices they don't + /// need. + pub fn fetchCandlesAndCorporateActions( self: *Tiingo, allocator: std.mem.Allocator, symbol: []const u8, from: Date, to: Date, - ) ![]Candle { + ) !CandleAndCorporateActions { var from_buf: [10]u8 = undefined; var to_buf: [10]u8 = undefined; const from_str = std.fmt.bufPrint(&from_buf, "{f}", .{from}) catch unreachable; @@ -60,13 +112,59 @@ pub const Tiingo = struct { var response = try self.client.get(url); defer response.deinit(); - return parseCandles(allocator, response.body); + return parseAll(allocator, response.body); + } + + /// Fetch daily candles for a symbol between two dates (inclusive). + /// Convenience wrapper around `fetchCandlesAndCorporateActions` + /// for callers that only want the candle slice. + pub fn fetchCandles( + self: *Tiingo, + allocator: std.mem.Allocator, + symbol: []const u8, + from: Date, + to: Date, + ) ![]Candle { + const triple = try self.fetchCandlesAndCorporateActions(allocator, symbol, from, to); + Dividend.freeSlice(allocator, triple.dividends); + allocator.free(triple.splits); + return triple.candles; + } + + /// Fetch dividends for a symbol between two dates (inclusive). + /// Convenience wrapper around `fetchCandlesAndCorporateActions`. + pub fn fetchDividends( + self: *Tiingo, + allocator: std.mem.Allocator, + symbol: []const u8, + from: Date, + to: Date, + ) ![]Dividend { + const triple = try self.fetchCandlesAndCorporateActions(allocator, symbol, from, to); + allocator.free(triple.candles); + allocator.free(triple.splits); + return triple.dividends; + } + + /// Fetch splits for a symbol between two dates (inclusive). + /// Convenience wrapper around `fetchCandlesAndCorporateActions`. + pub fn fetchSplits( + self: *Tiingo, + allocator: std.mem.Allocator, + symbol: []const u8, + from: Date, + to: Date, + ) ![]Split { + const triple = try self.fetchCandlesAndCorporateActions(allocator, symbol, from, to); + allocator.free(triple.candles); + Dividend.freeSlice(allocator, triple.dividends); + return triple.splits; } }; -/// Parse Tiingo's JSON array of price objects into Candles. -/// Tiingo returns oldest-first, which matches our convention. -fn parseCandles(allocator: std.mem.Allocator, body: []const u8) ![]Candle { +/// Walk Tiingo's JSON array of price rows once, emitting candles, +/// dividends (where `divCash != 0`), and splits (where `splitFactor != 1`). +fn parseAll(allocator: std.mem.Allocator, body: []const u8) !CandleAndCorporateActions { const parsed = std.json.parseFromSlice(std.json.Value, allocator, body, .{}) catch return error.ParseError; defer parsed.deinit(); @@ -78,10 +176,17 @@ fn parseCandles(allocator: std.mem.Allocator, body: []const u8) ![]Candle { }; var candles: std.ArrayList(Candle) = .empty; + errdefer candles.deinit(allocator); + + var dividends: std.ArrayList(Dividend) = .empty; errdefer { - candles.deinit(allocator); + for (dividends.items) |d| d.deinit(allocator); + dividends.deinit(allocator); } + var splits: std.ArrayList(Split) = .empty; + errdefer splits.deinit(allocator); + for (items) |item| { const obj = switch (item) { .object => |o| o, @@ -103,9 +208,40 @@ fn parseCandles(allocator: std.mem.Allocator, body: []const u8) ![]Candle { break :blk @intFromFloat(@max(0, v)); }, }); + + // Dividend event on this row (if any) + const div_cash = optFloat(obj.get("divCash")) orelse 0; + if (div_cash != 0) { + try dividends.append(allocator, .{ + .ex_date = date, + .amount = div_cash, + // Tiingo doesn't carry pay_date / record_date / + // frequency / type. Display-only fields stay null / + // .unknown; total-return math only needs ex_date and + // amount. + }); + } + + // Split event on this row (if any). Tiingo represents a 4:1 + // split as splitFactor = 4.0 and a 1:10 reverse split as + // splitFactor = 0.1. Both shapes are stored as + // numerator=splitFactor, denominator=1.0; `Split.ratio()` + // returns splitFactor in either case. + const split_factor = optFloat(obj.get("splitFactor")) orelse 1.0; + if (split_factor != 1.0 and split_factor != 0) { + try splits.append(allocator, .{ + .date = date, + .numerator = split_factor, + .denominator = 1.0, + }); + } } - return candles.toOwnedSlice(allocator); + return .{ + .candles = try candles.toOwnedSlice(allocator), + .dividends = try dividends.toOwnedSlice(allocator), + .splits = try splits.toOwnedSlice(allocator), + }; } /// Parse a Tiingo date string (e.g. "2026-03-16T00:00:00.000Z") into a Date. @@ -117,7 +253,7 @@ fn parseDate(val: ?std.json.Value) ?Date { // -- Tests -- -test "parseCandles basic" { +test "parseAll basic candles, no events" { const body = \\[ \\ { @@ -138,61 +274,188 @@ test "parseCandles basic" { ; const allocator = std.testing.allocator; - const candles = try parseCandles(allocator, body); - defer allocator.free(candles); + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); - try std.testing.expectEqual(@as(usize, 2), candles.len); + try std.testing.expectEqual(@as(usize, 2), triple.candles.len); + try std.testing.expectEqual(@as(usize, 0), triple.dividends.len); + try std.testing.expectEqual(@as(usize, 0), triple.splits.len); - // Oldest first - try std.testing.expectEqual(@as(i16, 2026), candles[0].date.year()); - try std.testing.expectEqual(@as(u8, 3), candles[0].date.month()); - try std.testing.expectEqual(@as(u8, 13), candles[0].date.day()); - try std.testing.expectApproxEqAbs(@as(f64, 42.41), candles[0].close, 0.01); - - try std.testing.expectEqual(@as(u8, 16), candles[1].date.day()); - try std.testing.expectApproxEqAbs(@as(f64, 42.74), candles[1].close, 0.01); - try std.testing.expectApproxEqAbs(@as(f64, 42.74), candles[1].adj_close, 0.01); + try std.testing.expectEqual(@as(i16, 2026), triple.candles[0].date.year()); + try std.testing.expectApproxEqAbs(@as(f64, 42.41), triple.candles[0].close, 0.01); + try std.testing.expectApproxEqAbs(@as(f64, 42.74), triple.candles[1].close, 0.01); } -test "parseCandles stock with volume" { +test "parseAll extracts a dividend from a divCash row" { + // NKE 2024-03-01 dividend of $0.37 (real Tiingo response shape) const body = \\[ \\ { - \\ "date": "2026-03-16T00:00:00.000Z", - \\ "close": 183.22, "high": 185.10, "low": 180.50, "open": 181.00, - \\ "volume": 217307380, "adjClose": 183.22, "adjHigh": 185.10, - \\ "adjLow": 180.50, "adjOpen": 181.00, "adjVolume": 217307380, + \\ "date": "2024-03-01T00:00:00.000Z", + \\ "close": 101.88, "high": 103.94, "low": 101.83, "open": 103.87, + \\ "volume": 7349270, "adjClose": 97.5550917628, + \\ "divCash": 0.37, "splitFactor": 1.0 + \\ } + \\] + ; + + const allocator = std.testing.allocator; + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); + + try std.testing.expectEqual(@as(usize, 1), triple.candles.len); + try std.testing.expectEqual(@as(usize, 1), triple.dividends.len); + try std.testing.expectEqual(@as(usize, 0), triple.splits.len); + + const div = triple.dividends[0]; + try std.testing.expect(div.ex_date.eql(Date.fromYmd(2024, 3, 1))); + try std.testing.expectApproxEqAbs(@as(f64, 0.37), div.amount, 0.001); + // Metadata fields are absent for Tiingo-sourced dividends + try std.testing.expect(div.pay_date == null); + try std.testing.expect(div.record_date == null); + try std.testing.expect(div.frequency == null); + try std.testing.expectEqual(@import("../models/dividend.zig").DividendType.unknown, div.type); +} + +test "parseAll extracts forward 4:1 split (SPYM 2017 fixture)" { + // SPYM's actual 2017-10-16 split — verbatim Tiingo response shape. + // Polygon and FMP both miss this split; Tiingo has it via + // splitFactor: 4.0. + const body = + \\[ + \\ { + \\ "date": "2017-10-13T00:00:00.000Z", + \\ "close": 119.7493, "open": 119.7667, "high": 120.26, "low": 119.7396, + \\ "volume": 7638, "adjClose": 26.0674934371, + \\ "divCash": 0.0, "splitFactor": 1.0 + \\ }, + \\ { + \\ "date": "2017-10-16T00:00:00.000Z", + \\ "close": 29.9556, "open": 30.01, "high": 30.01, "low": 29.9399, + \\ "volume": 8804, "adjClose": 26.0834061294, + \\ "divCash": 0.0, "splitFactor": 4.0 + \\ }, + \\ { + \\ "date": "2017-10-17T00:00:00.000Z", + \\ "close": 29.92, "open": 29.95, "high": 30.05, "low": 29.92, + \\ "volume": 21456, "adjClose": 26.0524079435, \\ "divCash": 0.0, "splitFactor": 1.0 \\ } \\] ; const allocator = std.testing.allocator; - const candles = try parseCandles(allocator, body); - defer allocator.free(candles); + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); - try std.testing.expectEqual(@as(usize, 1), candles.len); - try std.testing.expectApproxEqAbs(@as(f64, 181.00), candles[0].open, 0.01); - try std.testing.expectApproxEqAbs(@as(f64, 185.10), candles[0].high, 0.01); - try std.testing.expectApproxEqAbs(@as(f64, 180.50), candles[0].low, 0.01); - try std.testing.expectApproxEqAbs(@as(f64, 183.22), candles[0].close, 0.01); - try std.testing.expectEqual(@as(u64, 217307380), candles[0].volume); + try std.testing.expectEqual(@as(usize, 3), triple.candles.len); + try std.testing.expectEqual(@as(usize, 0), triple.dividends.len); + try std.testing.expectEqual(@as(usize, 1), triple.splits.len); + + const split = triple.splits[0]; + try std.testing.expect(split.date.eql(Date.fromYmd(2017, 10, 16))); + try std.testing.expectApproxEqAbs(@as(f64, 4.0), split.numerator, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 1.0), split.denominator, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 4.0), split.ratio(), 0.001); } -test "parseCandles error response" { +test "parseAll extracts reverse 1:10 split (splitFactor < 1)" { + // Reverse split: 1:10 means splitFactor = 0.1 + const body = + \\[ + \\ { + \\ "date": "2024-06-10T00:00:00.000Z", + \\ "close": 50.0, "open": 5.0, "high": 50.0, "low": 5.0, + \\ "volume": 1000, "adjClose": 50.0, + \\ "divCash": 0.0, "splitFactor": 0.1 + \\ } + \\] + ; + + const allocator = std.testing.allocator; + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); + + try std.testing.expectEqual(@as(usize, 1), triple.splits.len); + const split = triple.splits[0]; + try std.testing.expectApproxEqAbs(@as(f64, 0.1), split.ratio(), 0.001); +} + +test "parseAll: combined dividend + split in same response" { + const body = + \\[ + \\ {"date": "2024-01-15T00:00:00.000Z", "close": 100.0, "open": 100.0, "high": 100.0, "low": 100.0, + \\ "volume": 0, "adjClose": 100.0, "divCash": 0.5, "splitFactor": 1.0}, + \\ {"date": "2024-06-10T00:00:00.000Z", "close": 25.0, "open": 100.0, "high": 100.0, "low": 25.0, + \\ "volume": 0, "adjClose": 25.0, "divCash": 0.0, "splitFactor": 4.0}, + \\ {"date": "2024-12-15T00:00:00.000Z", "close": 30.0, "open": 30.0, "high": 30.0, "low": 30.0, + \\ "volume": 0, "adjClose": 30.0, "divCash": 0.15, "splitFactor": 1.0} + \\] + ; + + const allocator = std.testing.allocator; + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); + + try std.testing.expectEqual(@as(usize, 3), triple.candles.len); + try std.testing.expectEqual(@as(usize, 2), triple.dividends.len); + try std.testing.expectEqual(@as(usize, 1), triple.splits.len); + + try std.testing.expectApproxEqAbs(@as(f64, 0.5), triple.dividends[0].amount, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 0.15), triple.dividends[1].amount, 0.001); + try std.testing.expectApproxEqAbs(@as(f64, 4.0), triple.splits[0].ratio(), 0.001); +} + +test "parseAll: large dividend (VPMAX-style cap-gains distribution)" { + // VPMAX's 2025-12-17 distribution of $30.43 — chunky year-end + // cap-gains payout that inflates 1Y total return because Tiingo + // (and Polygon) lump it under regular dividends. + const body = + \\[ + \\ {"date": "2025-12-17T00:00:00.000Z", "close": 214.0, "open": 244.43, "high": 244.43, "low": 213.5, + \\ "volume": 0, "adjClose": 214.0, "divCash": 30.429903, "splitFactor": 1.0} + \\] + ; + + const allocator = std.testing.allocator; + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); + + try std.testing.expectEqual(@as(usize, 1), triple.dividends.len); + try std.testing.expectApproxEqAbs(@as(f64, 30.429903), triple.dividends[0].amount, 0.000001); +} + +test "parseAll error response" { const body = \\{"detail": "Not found."} ; const allocator = std.testing.allocator; - const result = parseCandles(allocator, body); + const result = parseAll(allocator, body); try std.testing.expectError(error.RequestFailed, result); } -test "parseCandles empty array" { +test "parseAll empty array" { const body = "[]"; const allocator = std.testing.allocator; - const candles = try parseCandles(allocator, body); - defer allocator.free(candles); - try std.testing.expectEqual(@as(usize, 0), candles.len); + const triple = try parseAll(allocator, body); + defer allocator.free(triple.candles); + defer Dividend.freeSlice(allocator, triple.dividends); + defer allocator.free(triple.splits); + + try std.testing.expectEqual(@as(usize, 0), triple.candles.len); + try std.testing.expectEqual(@as(usize, 0), triple.dividends.len); + try std.testing.expectEqual(@as(usize, 0), triple.splits.len); } diff --git a/src/service.zig b/src/service.zig index eda1a56..c37e770 100644 --- a/src/service.zig +++ b/src/service.zig @@ -352,10 +352,23 @@ pub const DataService = struct { fn fetchFromProvider(self: *DataService, comptime T: type, symbol: []const u8) !cache.Store.DataFor(T) { return switch (T) { Dividend => { + // Polygon is the primary source: it carries + // forward-looking declared dividends (e.g. ARCC's + // 2026-06-15 ex_date), which Tiingo's price-series + // response does not. Tiingo opportunistically + // supplements the cache via `populateAllFromTiingo` + // when candle fetches happen — that path uses the + // sorted-union write semantics in + // `cache.Store.writeMerged`, so Polygon's entries + // and Tiingo's entries coexist in `dividends.srf` + // without overwriting each other. var pg = try self.getProvider(Polygon); return pg.fetchDividends(self.allocator, symbol, null, null); }, Split => { + // Same rationale as Dividend above. Polygon also + // carries forward-looking split announcements that + // Tiingo's price-series doesn't surface. var pg = try self.getProvider(Polygon); return pg.fetchSplits(self.allocator, symbol); }, @@ -367,6 +380,57 @@ pub const DataService = struct { }; } + /// Fetch candles, dividends, and splits from Tiingo in a single + /// HTTP call and write all three caches. Returns the triple so + /// the caller can use the data without re-reading from disk. + /// + /// This is the orchestrated "cold cache" path. `getCandles` + /// (cold-cache full fetch) calls this so a single Tiingo HTTP + /// request populates `candles_daily.srf`, `candles_meta.srf`, + /// `dividends.srf`, and `splits.srf` together. Tiingo's + /// per-row `divCash` and `splitFactor` make this almost free. + /// + /// For dividends and splits the writes go through + /// `writeWithSource` with `"tiingo"` as the source hint. The + /// underlying `writeMerged` primitive merges Tiingo's view + /// into whatever's already on disk (typically Polygon-sourced + /// records), preserving forward-looking entries Polygon + /// uniquely carries. New entries trigger an `info(cache)` log + /// line attributing the discovery to Tiingo — useful when + /// Tiingo surfaces a corporate action Polygon missed (the + /// canonical case is SPYM's 2017-10-16 4:1 split). + /// + /// `from` is fixed at 2000-01-01 to cover any 10Y trailing-return + /// window even when `--as-of` back-dates the reference to the + /// earliest imported portfolio data (currently 2014). The extra + /// few years of pre-2004 candles cost ~150 KB per symbol on disk + /// and a one-time bandwidth bump on cold-cache fetch, both + /// trivial. Also gives a comfortable buffer for older corporate + /// actions (e.g. SPYM's 2017-10-16 split, deep-history reverse + /// splits on legacy tickers). + fn populateAllFromTiingo(self: *DataService, symbol: []const u8) !@import("providers/tiingo.zig").CandleAndCorporateActions { + var tg = try self.getProvider(Tiingo); + const today = fmt.todayDate(self.io); + const from = Date.fromYmd(2000, 1, 1); + const triple = try tg.fetchCandlesAndCorporateActions(self.allocator, symbol, from, today); + + var s = self.store(); + // Candles + meta — `cacheCandles` writes both candles_daily.srf + // and candles_meta.srf in one shot (last_close, last_date, + // provider, fail_count=0). + if (triple.candles.len > 0) { + s.cacheCandles(symbol, triple.candles, .tiingo, 0); + } + // Dividends and splits use the merge write path so Tiingo's + // view supplements rather than replaces existing (typically + // Polygon-sourced) records. New entries are logged with + // "tiingo" attribution. + s.writeWithSource(Dividend, symbol, triple.dividends, cache.DataType.dividends.ttl(), "tiingo"); + s.writeWithSource(Split, symbol, triple.splits, cache.DataType.splits.ttl(), "tiingo"); + + return triple; + } + /// Invalidate cached data for a symbol so the next get* call forces a fresh fetch. pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void { var s = self.store(); @@ -574,33 +638,39 @@ pub const DataService = struct { log.debug("{s}: candles synced from server but stale, falling through to full fetch", .{symbol}); } - // No usable cache — full fetch (~10 years, plus buffer for leap years) + // No usable cache — full fetch via the orchestrated Tiingo + // helper, which writes candles + dividends + splits caches in + // one shot from a single HTTP response. The fixed start date + // (see `populateAllFromTiingo`) is 2000-01-01, deep enough to + // cover a 10Y trailing-return window even when `--as-of` + // back-dates the reference into 2014-era imported portfolio + // history, plus a buffer for older corporate actions like + // SPYM's 2017-10-16 split. log.debug("{s}: fetching full candle history from provider", .{symbol}); - const from = today.addDays(-3700); - const result = self.fetchCandlesFromProviders(symbol, from, today, .tiingo) catch |err| { - if (err == DataError.TransientError) { - // On a fresh fetch, increment fail_count if we have meta + const triple = self.populateAllFromTiingo(symbol) catch |err| { + if (err == error.RateLimited or err == error.ServerError or err == error.RequestFailed) { + // Transient: increment fail_count on existing meta so + // we know to back off if this keeps happening. if (meta_result) |mr| { const new_fail_count = mr.meta.fail_count +| 1; s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count); } return DataError.TransientError; } - // FetchFailed at this point means BOTH Tiingo and Yahoo - // returned NotFound (or Yahoo was unavailable on top of - // Tiingo NotFound) — symbol genuinely has no candle data - // anywhere we look. Negative-cache the result so we don't - // keep retrying nonexistent symbols. + // NotFound, ParseError, InvalidResponse, AuthError — + // symbol genuinely has no candle data on Tiingo (the only + // provider for historical candles since the 2026-05 + // audit). Negative-cache so we don't keep retrying. s.writeNegative(symbol, .candles_daily); return DataError.FetchFailed; }; + // populateAllFromTiingo writes all three caches itself; we + // free the slices we don't return. + defer Dividend.freeSlice(self.allocator, triple.dividends); + defer self.allocator.free(triple.splits); - if (result.candles.len > 0) { - s.cacheCandles(symbol, result.candles, result.provider, 0); // reset fail_count on success - } - - return .{ .data = result.candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; + return .{ .data = triple.candles, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } /// Fetch dividend history for a symbol.